2024-11-25 10:55:00,601 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 10:55:00,631 main DEBUG Took 0.025905 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 10:55:00,632 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 10:55:00,633 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 10:55:00,637 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 10:55:00,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,650 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 10:55:00,674 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,677 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,679 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,682 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,683 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,685 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,686 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,687 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,688 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,689 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,690 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,690 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,691 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 10:55:00,692 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,692 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 10:55:00,694 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 10:55:00,696 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 10:55:00,698 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 10:55:00,699 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 10:55:00,700 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 10:55:00,701 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 10:55:00,717 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 10:55:00,722 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 10:55:00,724 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 10:55:00,725 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 10:55:00,725 main DEBUG createAppenders(={Console}) 2024-11-25 10:55:00,726 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-25 10:55:00,727 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 10:55:00,727 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-25 10:55:00,728 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 10:55:00,728 main DEBUG OutputStream closed 2024-11-25 10:55:00,729 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 10:55:00,730 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 10:55:00,730 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-25 10:55:00,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 10:55:00,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 10:55:00,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 10:55:00,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 10:55:00,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 10:55:00,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 10:55:00,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 10:55:00,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 10:55:00,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 10:55:00,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 10:55:00,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 10:55:00,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 10:55:00,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 10:55:00,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 10:55:00,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 10:55:00,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 10:55:00,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 10:55:00,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 10:55:00,985 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 10:55:00,986 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-25 10:55:00,986 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 10:55:00,987 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-25T10:55:01,602 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253 2024-11-25 10:55:01,613 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 10:55:01,615 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T10:55:01,647 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-25T10:55:01,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T10:55:01,758 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009, deleteOnExit=true 2024-11-25T10:55:01,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T10:55:01,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/test.cache.data in system properties and HBase conf 2024-11-25T10:55:01,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T10:55:01,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/hadoop.log.dir in system properties and HBase conf 2024-11-25T10:55:01,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T10:55:01,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T10:55:01,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T10:55:02,082 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T10:55:02,309 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T10:55:02,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T10:55:02,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T10:55:02,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T10:55:02,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T10:55:02,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T10:55:02,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T10:55:02,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T10:55:02,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T10:55:02,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T10:55:02,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/nfs.dump.dir in system properties and HBase conf 2024-11-25T10:55:02,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/java.io.tmpdir in system properties and HBase conf 2024-11-25T10:55:02,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T10:55:02,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T10:55:02,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T10:55:04,096 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T10:55:04,245 INFO [Time-limited test {}] log.Log(170): Logging initialized @5400ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T10:55:04,421 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T10:55:04,577 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T10:55:04,716 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T10:55:04,718 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T10:55:04,721 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T10:55:04,769 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T10:55:04,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/hadoop.log.dir/,AVAILABLE} 2024-11-25T10:55:04,779 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T10:55:05,081 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/java.io.tmpdir/jetty-localhost-36321-hadoop-hdfs-3_4_1-tests_jar-_-any-11191241275117520007/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T10:55:05,136 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:36321} 2024-11-25T10:55:05,136 INFO [Time-limited test {}] server.Server(415): Started @6293ms 2024-11-25T10:55:05,879 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T10:55:05,887 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T10:55:05,888 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T10:55:05,889 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T10:55:05,889 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T10:55:05,893 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/hadoop.log.dir/,AVAILABLE} 2024-11-25T10:55:05,894 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T10:55:06,048 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/java.io.tmpdir/jetty-localhost-44607-hadoop-hdfs-3_4_1-tests_jar-_-any-14044978367156114861/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T10:55:06,050 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:44607} 2024-11-25T10:55:06,051 INFO [Time-limited test {}] server.Server(415): Started @7207ms 2024-11-25T10:55:06,143 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T10:55:07,143 WARN [Thread-69 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/data/data1/current/BP-161473846-172.17.0.3-1732532103561/current, will proceed with Du for space computation calculation, 2024-11-25T10:55:07,144 WARN [Thread-70 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/data/data2/current/BP-161473846-172.17.0.3-1732532103561/current, will proceed with Du for space computation calculation, 2024-11-25T10:55:07,336 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T10:55:07,483 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253 2024-11-25T10:55:07,520 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c92cbf9b7cf8852 with lease ID 0x6d7fc9d91aaaa04f: Processing first storage report for DS-54bddccd-0be7-4fd1-80f3-c7647f733a52 from datanode DatanodeRegistration(127.0.0.1:37429, datanodeUuid=2cdee2b9-c7d3-4ba4-9102-a7bc63c6d504, infoPort=40281, infoSecurePort=0, ipcPort=32915, storageInfo=lv=-57;cid=testClusterID;nsid=527707933;c=1732532103561) 2024-11-25T10:55:07,522 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c92cbf9b7cf8852 with lease ID 0x6d7fc9d91aaaa04f: from storage DS-54bddccd-0be7-4fd1-80f3-c7647f733a52 node DatanodeRegistration(127.0.0.1:37429, datanodeUuid=2cdee2b9-c7d3-4ba4-9102-a7bc63c6d504, infoPort=40281, infoSecurePort=0, ipcPort=32915, storageInfo=lv=-57;cid=testClusterID;nsid=527707933;c=1732532103561), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T10:55:07,522 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c92cbf9b7cf8852 with lease ID 0x6d7fc9d91aaaa04f: Processing first storage report for DS-954c955e-1d8d-4827-a5d5-97a31db64224 from datanode DatanodeRegistration(127.0.0.1:37429, datanodeUuid=2cdee2b9-c7d3-4ba4-9102-a7bc63c6d504, infoPort=40281, infoSecurePort=0, ipcPort=32915, storageInfo=lv=-57;cid=testClusterID;nsid=527707933;c=1732532103561) 2024-11-25T10:55:07,523 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c92cbf9b7cf8852 with lease ID 0x6d7fc9d91aaaa04f: from storage DS-954c955e-1d8d-4827-a5d5-97a31db64224 node DatanodeRegistration(127.0.0.1:37429, datanodeUuid=2cdee2b9-c7d3-4ba4-9102-a7bc63c6d504, infoPort=40281, infoSecurePort=0, ipcPort=32915, storageInfo=lv=-57;cid=testClusterID;nsid=527707933;c=1732532103561), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T10:55:07,743 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/zookeeper_0, clientPort=57296, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T10:55:07,777 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57296 2024-11-25T10:55:07,797 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T10:55:07,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T10:55:08,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741825_1001 (size=7) 2024-11-25T10:55:08,824 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 with version=8 2024-11-25T10:55:08,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/hbase-staging 2024-11-25T10:55:08,996 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T10:55:09,371 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T10:55:09,387 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T10:55:09,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T10:55:09,397 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T10:55:09,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T10:55:09,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T10:55:09,660 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T10:55:09,785 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T10:55:09,805 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T10:55:09,812 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T10:55:09,862 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1628 (auto-detected) 2024-11-25T10:55:09,865 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-25T10:55:09,912 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39013 2024-11-25T10:55:09,951 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39013 connecting to ZooKeeper ensemble=127.0.0.1:57296 2024-11-25T10:55:10,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390130x0, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T10:55:10,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39013-0x10076940c1b0000 connected 2024-11-25T10:55:10,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T10:55:10,185 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T10:55:10,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T10:55:10,214 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770, hbase.cluster.distributed=false 2024-11-25T10:55:10,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T10:55:10,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39013 2024-11-25T10:55:10,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39013 2024-11-25T10:55:10,297 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39013 2024-11-25T10:55:10,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39013 2024-11-25T10:55:10,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39013 2024-11-25T10:55:10,461 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T10:55:10,464 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T10:55:10,464 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T10:55:10,465 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T10:55:10,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T10:55:10,465 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T10:55:10,469 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T10:55:10,477 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T10:55:10,479 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44671 2024-11-25T10:55:10,483 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44671 connecting to ZooKeeper ensemble=127.0.0.1:57296 2024-11-25T10:55:10,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T10:55:10,495 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T10:55:10,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446710x0, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T10:55:10,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44671-0x10076940c1b0001 connected 2024-11-25T10:55:10,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T10:55:10,538 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T10:55:10,563 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T10:55:10,570 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T10:55:10,583 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T10:55:10,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44671 2024-11-25T10:55:10,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44671 2024-11-25T10:55:10,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44671 2024-11-25T10:55:10,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44671 2024-11-25T10:55:10,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44671 2024-11-25T10:55:10,653 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6bfc37bad115:39013 2024-11-25T10:55:10,655 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6bfc37bad115,39013,1732532109078 2024-11-25T10:55:10,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T10:55:10,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T10:55:10,672 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6bfc37bad115,39013,1732532109078 2024-11-25T10:55:10,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T10:55:10,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:10,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:10,732 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T10:55:10,737 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6bfc37bad115,39013,1732532109078 from backup master directory 2024-11-25T10:55:10,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T10:55:10,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6bfc37bad115,39013,1732532109078 2024-11-25T10:55:10,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T10:55:10,748 WARN [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T10:55:10,748 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6bfc37bad115,39013,1732532109078 2024-11-25T10:55:10,750 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T10:55:10,752 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T10:55:10,846 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/hbase.id] with ID: d15501a3-200b-4410-9329-faccae53cbbc 2024-11-25T10:55:10,846 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/.tmp/hbase.id 2024-11-25T10:55:10,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741826_1002 (size=42) 2024-11-25T10:55:11,312 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/.tmp/hbase.id]:[hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/hbase.id] 2024-11-25T10:55:11,400 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T10:55:11,407 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T10:55:11,456 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 39ms. 2024-11-25T10:55:11,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:11,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:11,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741827_1003 (size=196) 2024-11-25T10:55:11,952 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:55:11,956 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T10:55:11,979 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:11,987 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T10:55:12,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741828_1004 (size=1189) 2024-11-25T10:55:12,497 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store 2024-11-25T10:55:12,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741829_1005 (size=34) 2024-11-25T10:55:12,549 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T10:55:12,554 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:12,556 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T10:55:12,556 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:55:12,556 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:55:12,559 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T10:55:12,559 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:55:12,559 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:55:12,561 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732532112556Disabling compacts and flushes for region at 1732532112556Disabling writes for close at 1732532112559 (+3 ms)Writing region close event to WAL at 1732532112559Closed at 1732532112559 2024-11-25T10:55:12,564 WARN [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/.initializing 2024-11-25T10:55:12,564 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/WALs/6bfc37bad115,39013,1732532109078 2024-11-25T10:55:12,581 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T10:55:12,612 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6bfc37bad115%2C39013%2C1732532109078, suffix=, logDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/WALs/6bfc37bad115,39013,1732532109078, archiveDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/oldWALs, maxLogs=10 2024-11-25T10:55:12,646 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/WALs/6bfc37bad115,39013,1732532109078/6bfc37bad115%2C39013%2C1732532109078.1732532112619, exclude list is [], retry=0 2024-11-25T10:55:12,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37429,DS-54bddccd-0be7-4fd1-80f3-c7647f733a52,DISK] 2024-11-25T10:55:12,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T10:55:12,740 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/WALs/6bfc37bad115,39013,1732532109078/6bfc37bad115%2C39013%2C1732532109078.1732532112619 2024-11-25T10:55:12,745 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40281:40281)] 2024-11-25T10:55:12,746 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:55:12,747 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:12,751 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,753 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T10:55:12,854 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:12,858 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:12,859 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T10:55:12,863 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:12,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:12,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T10:55:12,875 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:12,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:12,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T10:55:12,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:12,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:12,892 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,898 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,899 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,908 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,909 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,914 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T10:55:12,928 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T10:55:12,947 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:55:12,950 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72281693, jitterRate=0.07708115875720978}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T10:55:12,957 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732532112772Initializing all the Stores at 1732532112776 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532112776Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532112780 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532112781 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532112781Cleaning up temporary data from old regions at 1732532112909 (+128 ms)Region opened successfully at 1732532112957 (+48 ms) 2024-11-25T10:55:12,961 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T10:55:13,016 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735ba3a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T10:55:13,076 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T10:55:13,090 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T10:55:13,090 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T10:55:13,094 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T10:55:13,098 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-25T10:55:13,105 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-25T10:55:13,105 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T10:55:13,172 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T10:55:13,188 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T10:55:13,195 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T10:55:13,199 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T10:55:13,202 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T10:55:13,208 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T10:55:13,212 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T10:55:13,217 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T10:55:13,221 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T10:55:13,223 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T10:55:13,238 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T10:55:13,284 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T10:55:13,292 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T10:55:13,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T10:55:13,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T10:55:13,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:13,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:13,309 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6bfc37bad115,39013,1732532109078, sessionid=0x10076940c1b0000, setting cluster-up flag (Was=false) 2024-11-25T10:55:13,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:13,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:13,339 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T10:55:13,342 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6bfc37bad115,39013,1732532109078 2024-11-25T10:55:13,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:13,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:13,362 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T10:55:13,368 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6bfc37bad115,39013,1732532109078 2024-11-25T10:55:13,403 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T10:55:13,445 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(746): ClusterId : d15501a3-200b-4410-9329-faccae53cbbc 2024-11-25T10:55:13,448 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T10:55:13,465 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T10:55:13,465 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T10:55:13,476 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T10:55:13,477 DEBUG [RS:0;6bfc37bad115:44671 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6f90cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T10:55:13,509 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6bfc37bad115:44671 2024-11-25T10:55:13,516 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T10:55:13,517 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T10:55:13,517 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T10:55:13,520 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T10:55:13,521 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(2659): reportForDuty to master=6bfc37bad115,39013,1732532109078 with port=44671, startcode=1732532110409 2024-11-25T10:55:13,544 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T10:55:13,545 DEBUG [RS:0;6bfc37bad115:44671 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T10:55:13,554 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T10:55:13,561 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6bfc37bad115,39013,1732532109078 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T10:55:13,582 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T10:55:13,583 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T10:55:13,584 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T10:55:13,584 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T10:55:13,584 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6bfc37bad115:0, corePoolSize=10, maxPoolSize=10 2024-11-25T10:55:13,584 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,585 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T10:55:13,585 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,595 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T10:55:13,596 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T10:55:13,608 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:13,608 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T10:55:13,612 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732532143612 2024-11-25T10:55:13,614 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47091, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T10:55:13,615 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T10:55:13,616 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T10:55:13,623 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39013 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:13,623 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T10:55:13,624 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T10:55:13,624 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T10:55:13,625 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T10:55:13,633 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,637 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T10:55:13,638 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T10:55:13,639 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T10:55:13,644 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T10:55:13,645 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T10:55:13,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741831_1007 (size=1321) 2024-11-25T10:55:13,654 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T10:55:13,655 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:55:13,661 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532113646,5,FailOnTimeoutGroup] 2024-11-25T10:55:13,671 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T10:55:13,671 WARN [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T10:55:13,672 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532113664,5,FailOnTimeoutGroup] 2024-11-25T10:55:13,673 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,673 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T10:55:13,675 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,675 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741832_1008 (size=32) 2024-11-25T10:55:13,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:13,757 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T10:55:13,778 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(2659): reportForDuty to master=6bfc37bad115,39013,1732532109078 with port=44671, startcode=1732532110409 2024-11-25T10:55:13,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T10:55:13,780 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:13,782 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39013 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6bfc37bad115,44671,1732532110409 2024-11-25T10:55:13,786 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39013 {}] master.ServerManager(517): Registering regionserver=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:13,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:13,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T10:55:13,802 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T10:55:13,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:13,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:13,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T10:55:13,808 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:55:13,808 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36023 2024-11-25T10:55:13,808 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T10:55:13,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T10:55:13,815 DEBUG [RS:0;6bfc37bad115:44671 {}] zookeeper.ZKUtil(111): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6bfc37bad115,44671,1732532110409 2024-11-25T10:55:13,815 WARN [RS:0;6bfc37bad115:44671 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T10:55:13,815 INFO [RS:0;6bfc37bad115:44671 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T10:55:13,816 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/WALs/6bfc37bad115,44671,1732532110409 2024-11-25T10:55:13,821 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T10:55:13,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:13,830 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6bfc37bad115,44671,1732532110409] 2024-11-25T10:55:13,830 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:13,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T10:55:13,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T10:55:13,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:13,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:13,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T10:55:13,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740 2024-11-25T10:55:13,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740 2024-11-25T10:55:13,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T10:55:13,866 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T10:55:13,868 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:55:13,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T10:55:13,888 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T10:55:13,915 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:55:13,917 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T10:55:13,921 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72700754, jitterRate=0.08332565426826477}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:55:13,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732532113728Initializing all the Stores at 1732532113752 (+24 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532113753 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532113756 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532113756Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532113756Cleaning up temporary data from old regions at 1732532113866 (+110 ms)Region opened successfully at 1732532113924 (+58 ms) 2024-11-25T10:55:13,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T10:55:13,925 INFO [RS:0;6bfc37bad115:44671 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T10:55:13,925 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T10:55:13,925 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T10:55:13,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T10:55:13,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T10:55:13,929 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T10:55:13,937 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T10:55:13,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732532113925Disabling compacts and flushes for region at 1732532113925Disabling writes for close at 1732532113925Writing region close event to WAL at 1732532113937 (+12 ms)Closed at 1732532113937 2024-11-25T10:55:13,941 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T10:55:13,943 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T10:55:13,943 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T10:55:13,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T10:55:13,957 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,958 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,958 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,958 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,958 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,959 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,959 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T10:55:13,959 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,959 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,959 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,960 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,960 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,960 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T10:55:13,960 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T10:55:13,961 DEBUG [RS:0;6bfc37bad115:44671 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T10:55:13,968 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,969 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,969 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,969 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,969 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:13,969 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,44671,1732532110409-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T10:55:13,971 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T10:55:13,990 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T10:55:14,006 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T10:55:14,009 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,44671,1732532110409-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:14,009 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:14,010 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.Replication(171): 6bfc37bad115,44671,1732532110409 started 2024-11-25T10:55:14,060 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:14,068 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1482): Serving as 6bfc37bad115,44671,1732532110409, RpcServer on 6bfc37bad115/172.17.0.3:44671, sessionid=0x10076940c1b0001 2024-11-25T10:55:14,069 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T10:55:14,070 DEBUG [RS:0;6bfc37bad115:44671 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6bfc37bad115,44671,1732532110409 2024-11-25T10:55:14,070 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,44671,1732532110409' 2024-11-25T10:55:14,070 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T10:55:14,073 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T10:55:14,077 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T10:55:14,077 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T10:55:14,077 DEBUG [RS:0;6bfc37bad115:44671 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6bfc37bad115,44671,1732532110409 2024-11-25T10:55:14,078 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,44671,1732532110409' 2024-11-25T10:55:14,078 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T10:55:14,079 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T10:55:14,080 DEBUG [RS:0;6bfc37bad115:44671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T10:55:14,080 INFO [RS:0;6bfc37bad115:44671 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T10:55:14,080 INFO [RS:0;6bfc37bad115:44671 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T10:55:14,145 WARN [6bfc37bad115:39013 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T10:55:14,188 INFO [RS:0;6bfc37bad115:44671 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T10:55:14,193 INFO [RS:0;6bfc37bad115:44671 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6bfc37bad115%2C44671%2C1732532110409, suffix=, logDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/WALs/6bfc37bad115,44671,1732532110409, archiveDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/oldWALs, maxLogs=32 2024-11-25T10:55:14,253 DEBUG [RS:0;6bfc37bad115:44671 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/WALs/6bfc37bad115,44671,1732532110409/6bfc37bad115%2C44671%2C1732532110409.1732532114199, exclude list is [], retry=0 2024-11-25T10:55:14,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37429,DS-54bddccd-0be7-4fd1-80f3-c7647f733a52,DISK] 2024-11-25T10:55:14,311 INFO [RS:0;6bfc37bad115:44671 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/WALs/6bfc37bad115,44671,1732532110409/6bfc37bad115%2C44671%2C1732532110409.1732532114199 2024-11-25T10:55:14,314 DEBUG [RS:0;6bfc37bad115:44671 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40281:40281)] 2024-11-25T10:55:14,400 DEBUG [6bfc37bad115:39013 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T10:55:14,429 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:14,451 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,44671,1732532110409, state=OPENING 2024-11-25T10:55:14,469 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T10:55:14,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:14,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:55:14,489 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T10:55:14,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T10:55:14,491 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T10:55:14,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:55:14,697 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T10:55:14,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57195, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T10:55:14,721 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T10:55:14,721 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T10:55:14,722 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T10:55:14,727 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6bfc37bad115%2C44671%2C1732532110409.meta, suffix=.meta, logDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/WALs/6bfc37bad115,44671,1732532110409, archiveDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/oldWALs, maxLogs=32 2024-11-25T10:55:14,754 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/WALs/6bfc37bad115,44671,1732532110409/6bfc37bad115%2C44671%2C1732532110409.meta.1732532114730.meta, exclude list is [], retry=0 2024-11-25T10:55:14,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37429,DS-54bddccd-0be7-4fd1-80f3-c7647f733a52,DISK] 2024-11-25T10:55:14,769 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/WALs/6bfc37bad115,44671,1732532110409/6bfc37bad115%2C44671%2C1732532110409.meta.1732532114730.meta 2024-11-25T10:55:14,770 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40281:40281)] 2024-11-25T10:55:14,771 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:55:14,773 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T10:55:14,777 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T10:55:14,783 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T10:55:14,790 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T10:55:14,790 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:14,791 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T10:55:14,791 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T10:55:14,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T10:55:14,804 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T10:55:14,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:14,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:14,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T10:55:14,808 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T10:55:14,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:14,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:14,810 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T10:55:14,814 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T10:55:14,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:14,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:14,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T10:55:14,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T10:55:14,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:14,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T10:55:14,823 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T10:55:14,837 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740 2024-11-25T10:55:14,841 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740 2024-11-25T10:55:14,846 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T10:55:14,846 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T10:55:14,847 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:55:14,851 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T10:55:14,854 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59087614, jitterRate=-0.11952593922615051}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:55:14,854 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T10:55:14,855 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732532114792Writing region info on filesystem at 1732532114792Initializing all the Stores at 1732532114795 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532114795Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532114797 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532114797Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532114797Cleaning up temporary data from old regions at 1732532114846 (+49 ms)Running coprocessor post-open hooks at 1732532114854 (+8 ms)Region opened successfully at 1732532114855 (+1 ms) 2024-11-25T10:55:14,864 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732532114684 2024-11-25T10:55:14,888 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T10:55:14,889 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:14,889 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T10:55:14,892 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,44671,1732532110409, state=OPEN 2024-11-25T10:55:14,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T10:55:14,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T10:55:14,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T10:55:14,898 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T10:55:14,900 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:14,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T10:55:14,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,44671,1732532110409 in 407 msec 2024-11-25T10:55:14,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T10:55:14,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 961 msec 2024-11-25T10:55:14,931 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T10:55:14,931 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T10:55:14,970 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:14,971 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:15,007 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:15,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:15,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6150 sec 2024-11-25T10:55:15,069 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732532115068, completionTime=-1 2024-11-25T10:55:15,072 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T10:55:15,072 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T10:55:15,106 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T10:55:15,106 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732532175106 2024-11-25T10:55:15,106 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732532235106 2024-11-25T10:55:15,106 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-25T10:55:15,110 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39013,1732532109078-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:15,110 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39013,1732532109078-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:15,110 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39013,1732532109078-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:15,112 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6bfc37bad115:39013, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:15,113 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:15,115 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:15,134 DEBUG [master/6bfc37bad115:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T10:55:15,197 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.449sec 2024-11-25T10:55:15,199 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T10:55:15,207 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T10:55:15,209 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T10:55:15,210 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T10:55:15,210 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T10:55:15,212 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39013,1732532109078-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T10:55:15,213 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39013,1732532109078-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T10:55:15,288 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T10:55:15,290 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T10:55:15,290 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39013,1732532109078-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T10:55:15,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dfad15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:15,301 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T10:55:15,302 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T10:55:15,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:15,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:15,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:15,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67531474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:15,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:15,350 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:15,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b5c665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:15,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:15,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:15,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6bfc37bad115,39013,1732532109078 2024-11-25T10:55:15,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1017, ProcessCount=12, AvailableMemoryMB=731 2024-11-25T10:55:15,462 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T10:55:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39013,1732532109078 2024-11-25T10:55:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@797bf85d 2024-11-25T10:55:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T10:55:15,479 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T10:55:15,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:55:15,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:55:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:15,501 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T10:55:15,523 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:15,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-25T10:55:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T10:55:15,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T10:55:15,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741835_1011 (size=1061) 2024-11-25T10:55:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T10:55:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T10:55:16,003 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:55:16,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741836_1012 (size=53) 2024-11-25T10:55:16,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:16,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 619a47d3e2a0b4c567d3fd0e3f1893d1, disabling compactions & flushes 2024-11-25T10:55:16,031 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. after waiting 0 ms 2024-11-25T10:55:16,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,031 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: Waiting for close lock at 1732532116031Disabling compacts and flushes for region at 1732532116031Disabling writes for close at 1732532116031Writing region close event to WAL at 1732532116031Closed at 1732532116031 2024-11-25T10:55:16,036 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T10:55:16,043 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732532116036"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532116036"}]},"ts":"1732532116036"} 2024-11-25T10:55:16,049 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T10:55:16,052 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T10:55:16,056 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532116052"}]},"ts":"1732532116052"} 2024-11-25T10:55:16,062 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T10:55:16,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, ASSIGN}] 2024-11-25T10:55:16,069 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, ASSIGN 2024-11-25T10:55:16,071 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, ASSIGN; state=OFFLINE, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=false 2024-11-25T10:55:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T10:55:16,223 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=619a47d3e2a0b4c567d3fd0e3f1893d1, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:16,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, ASSIGN because future has completed 2024-11-25T10:55:16,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:55:16,399 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,399 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:55:16,400 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,400 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:16,401 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,401 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,413 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,423 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:16,424 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619a47d3e2a0b4c567d3fd0e3f1893d1 columnFamilyName A 2024-11-25T10:55:16,424 DEBUG [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:16,426 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.HStore(327): Store=619a47d3e2a0b4c567d3fd0e3f1893d1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:16,427 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,432 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:16,433 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619a47d3e2a0b4c567d3fd0e3f1893d1 columnFamilyName B 2024-11-25T10:55:16,433 DEBUG [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:16,434 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.HStore(327): Store=619a47d3e2a0b4c567d3fd0e3f1893d1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:16,435 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,439 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:16,440 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 619a47d3e2a0b4c567d3fd0e3f1893d1 columnFamilyName C 2024-11-25T10:55:16,440 DEBUG [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:16,441 INFO [StoreOpener-619a47d3e2a0b4c567d3fd0e3f1893d1-1 {}] regionserver.HStore(327): Store=619a47d3e2a0b4c567d3fd0e3f1893d1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:16,442 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,442 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,449 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,450 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,457 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,457 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,459 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:55:16,463 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,473 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:55:16,474 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 619a47d3e2a0b4c567d3fd0e3f1893d1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59206912, jitterRate=-0.11774826049804688}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:55:16,475 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,476 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: Running coprocessor pre-open hook at 1732532116401Writing region info on filesystem at 1732532116401Initializing all the Stores at 1732532116406 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532116407 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532116408 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532116408Cleaning up temporary data from old regions at 1732532116457 (+49 ms)Running coprocessor post-open hooks at 1732532116475 (+18 ms)Region opened successfully at 1732532116476 (+1 ms) 2024-11-25T10:55:16,480 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., pid=6, masterSystemTime=1732532116388 2024-11-25T10:55:16,491 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=619a47d3e2a0b4c567d3fd0e3f1893d1, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:16,492 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,492 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:16,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:55:16,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T10:55:16,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 in 278 msec 2024-11-25T10:55:16,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T10:55:16,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, ASSIGN in 458 msec 2024-11-25T10:55:16,537 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T10:55:16,538 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532116537"}]},"ts":"1732532116537"} 2024-11-25T10:55:16,547 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T10:55:16,550 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T10:55:16,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.0600 sec 2024-11-25T10:55:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T10:55:16,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:16,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705975cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,691 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b1a558, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f8a5ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,711 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T10:55:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39013,1732532109078 2024-11-25T10:55:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@489d97cb 2024-11-25T10:55:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T10:55:16,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T10:55:16,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6023f6f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,728 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35afed03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,731 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4112bfce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,744 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6869e1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,751 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2e42a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246c200c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad2d2cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97f651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5320901e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,785 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53079d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,788 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7afc60b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ebb6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31600ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,805 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c8af47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,807 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@976002f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,814 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@433e9672, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,817 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70456a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:16,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:16,822 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7268a93f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:16,825 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d52ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:16,834 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,835 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:16,837 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,838 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,840 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,842 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,844 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,845 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-25T10:55:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T10:55:16,850 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,852 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,853 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,855 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:16,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,860 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,861 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:16,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,883 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:16,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:16,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:16,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:55:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T10:55:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:16,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T10:55:16,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:16,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:17,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:17,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:17,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T10:55:17,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a9f54d01422649ee973ad4f4825f9b2e is 50, key is test_row_0/A:col10/1732532116879/Put/seqid=0 2024-11-25T10:55:17,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:17,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:17,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741837_1013 (size=12001) 2024-11-25T10:55:17,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a9f54d01422649ee973ad4f4825f9b2e 2024-11-25T10:55:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532177322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532177313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532177336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532177345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532177345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532177454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T10:55:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532177457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532177478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532177488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532177495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/5d135bba5cc0447895a99682bd13a6f7 is 50, key is test_row_0/B:col10/1732532116879/Put/seqid=0 2024-11-25T10:55:17,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741838_1014 (size=12001) 2024-11-25T10:55:17,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:17,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:17,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532177689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532177685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532177704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532177715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532177717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:17,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:17,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:17,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/5d135bba5cc0447895a99682bd13a6f7 2024-11-25T10:55:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T10:55:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532178005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532178010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532178026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532178033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532178040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3a5bb642c84a44368ad414e8b177bf66 is 50, key is test_row_0/C:col10/1732532116879/Put/seqid=0 2024-11-25T10:55:18,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:18,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:18,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:18,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741839_1015 (size=12001) 2024-11-25T10:55:18,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3a5bb642c84a44368ad414e8b177bf66 2024-11-25T10:55:18,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a9f54d01422649ee973ad4f4825f9b2e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9f54d01422649ee973ad4f4825f9b2e 2024-11-25T10:55:18,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9f54d01422649ee973ad4f4825f9b2e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T10:55:18,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/5d135bba5cc0447895a99682bd13a6f7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/5d135bba5cc0447895a99682bd13a6f7 2024-11-25T10:55:18,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:18,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/5d135bba5cc0447895a99682bd13a6f7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T10:55:18,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:18,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3a5bb642c84a44368ad414e8b177bf66 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3a5bb642c84a44368ad414e8b177bf66 2024-11-25T10:55:18,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:18,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:18,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3a5bb642c84a44368ad414e8b177bf66, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T10:55:18,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1351ms, sequenceid=13, compaction requested=false 2024-11-25T10:55:18,321 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T10:55:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:18,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T10:55:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:18,439 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T10:55:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:18,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/f87ff97a1c294729b02f05b23f2586a3 is 50, key is test_row_0/A:col10/1732532117319/Put/seqid=0 2024-11-25T10:55:18,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741840_1016 (size=12001) 2024-11-25T10:55:18,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:18,585 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/f87ff97a1c294729b02f05b23f2586a3 2024-11-25T10:55:18,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/dafe5af42b3b4ead9bdc82648527f0b2 is 50, key is test_row_0/B:col10/1732532117319/Put/seqid=0 2024-11-25T10:55:18,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741841_1017 (size=12001) 2024-11-25T10:55:18,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532178662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532178728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532178740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532178754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532178758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532178865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532178865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532178895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532178905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532178914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T10:55:19,082 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/dafe5af42b3b4ead9bdc82648527f0b2 2024-11-25T10:55:19,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532179086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532179096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6525a836ce3c462fb766406a53a6f927 is 50, key is test_row_0/C:col10/1732532117319/Put/seqid=0 2024-11-25T10:55:19,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532179117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532179137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532179137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741842_1018 (size=12001) 2024-11-25T10:55:19,176 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6525a836ce3c462fb766406a53a6f927 2024-11-25T10:55:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/f87ff97a1c294729b02f05b23f2586a3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/f87ff97a1c294729b02f05b23f2586a3 2024-11-25T10:55:19,275 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/f87ff97a1c294729b02f05b23f2586a3, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T10:55:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/dafe5af42b3b4ead9bdc82648527f0b2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/dafe5af42b3b4ead9bdc82648527f0b2 2024-11-25T10:55:19,319 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/dafe5af42b3b4ead9bdc82648527f0b2, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T10:55:19,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6525a836ce3c462fb766406a53a6f927 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6525a836ce3c462fb766406a53a6f927 2024-11-25T10:55:19,341 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6525a836ce3c462fb766406a53a6f927, entries=150, sequenceid=33, filesize=11.7 K 2024-11-25T10:55:19,345 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 905ms, sequenceid=33, compaction requested=false 2024-11-25T10:55:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-25T10:55:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-25T10:55:19,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T10:55:19,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5200 sec 2024-11-25T10:55:19,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5710 sec 2024-11-25T10:55:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:19,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T10:55:19,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:19,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:19,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:19,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:19,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/492d8089c03e43ffa120de2d3f4a633d is 50, key is test_row_0/A:col10/1732532118720/Put/seqid=0 2024-11-25T10:55:19,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741843_1019 (size=14341) 2024-11-25T10:55:19,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/492d8089c03e43ffa120de2d3f4a633d 2024-11-25T10:55:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532179570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532179575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/538cfbea6c8749a683a9711ec1bf3499 is 50, key is test_row_0/B:col10/1732532118720/Put/seqid=0 2024-11-25T10:55:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532179577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532179579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532179602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741844_1020 (size=12001) 2024-11-25T10:55:19,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/538cfbea6c8749a683a9711ec1bf3499 2024-11-25T10:55:19,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/ebb41d13ec6c46f58e52d90abe5716ad is 50, key is test_row_0/C:col10/1732532118720/Put/seqid=0 2024-11-25T10:55:19,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532179718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532179718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532179719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532179719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741845_1021 (size=12001) 2024-11-25T10:55:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532179740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:19,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/ebb41d13ec6c46f58e52d90abe5716ad 2024-11-25T10:55:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:19,769 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T10:55:19,770 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T10:55:19,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/492d8089c03e43ffa120de2d3f4a633d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/492d8089c03e43ffa120de2d3f4a633d 2024-11-25T10:55:19,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T10:55:19,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T10:55:19,792 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T10:55:19,792 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T10:55:19,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T10:55:19,794 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T10:55:19,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/492d8089c03e43ffa120de2d3f4a633d, entries=200, sequenceid=47, filesize=14.0 K 2024-11-25T10:55:19,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/538cfbea6c8749a683a9711ec1bf3499 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/538cfbea6c8749a683a9711ec1bf3499 2024-11-25T10:55:19,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/538cfbea6c8749a683a9711ec1bf3499, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T10:55:19,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/ebb41d13ec6c46f58e52d90abe5716ad as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/ebb41d13ec6c46f58e52d90abe5716ad 2024-11-25T10:55:19,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/ebb41d13ec6c46f58e52d90abe5716ad, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T10:55:19,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 406ms, sequenceid=47, compaction requested=true 2024-11-25T10:55:19,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:19,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:19,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:19,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:19,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:19,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:55:19,874 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:19,878 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:19,881 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:19,890 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:19,890 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:19,891 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/5d135bba5cc0447895a99682bd13a6f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/dafe5af42b3b4ead9bdc82648527f0b2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/538cfbea6c8749a683a9711ec1bf3499] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=35.2 K 2024-11-25T10:55:19,894 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d135bba5cc0447895a99682bd13a6f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732532116879 2024-11-25T10:55:19,894 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:19,894 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:19,894 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:19,895 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9f54d01422649ee973ad4f4825f9b2e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/f87ff97a1c294729b02f05b23f2586a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/492d8089c03e43ffa120de2d3f4a633d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=37.4 K 2024-11-25T10:55:19,895 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f54d01422649ee973ad4f4825f9b2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732532116879 2024-11-25T10:55:19,896 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting dafe5af42b3b4ead9bdc82648527f0b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732532117318 2024-11-25T10:55:19,896 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f87ff97a1c294729b02f05b23f2586a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732532117318 2024-11-25T10:55:19,897 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492d8089c03e43ffa120de2d3f4a633d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532118611 2024-11-25T10:55:19,898 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 538cfbea6c8749a683a9711ec1bf3499, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532118611 2024-11-25T10:55:19,961 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#9 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:19,961 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#10 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:19,962 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/8816887003784638a5c2502333197019 is 50, key is test_row_0/A:col10/1732532118720/Put/seqid=0 2024-11-25T10:55:19,964 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7ca3016a8e7643278c3bb1c82c05f318 is 50, key is test_row_0/B:col10/1732532118720/Put/seqid=0 2024-11-25T10:55:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:19,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:55:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:19,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:20,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741846_1022 (size=12104) 2024-11-25T10:55:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741847_1023 (size=12104) 2024-11-25T10:55:20,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/986e4edbfb874e2ab467b243593051c9 is 50, key is test_row_0/A:col10/1732532119967/Put/seqid=0 2024-11-25T10:55:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532180022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,051 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/8816887003784638a5c2502333197019 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/8816887003784638a5c2502333197019 2024-11-25T10:55:20,056 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7ca3016a8e7643278c3bb1c82c05f318 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ca3016a8e7643278c3bb1c82c05f318 2024-11-25T10:55:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532180027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532180028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532180033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532180045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,087 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 7ca3016a8e7643278c3bb1c82c05f318(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:20,087 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 8816887003784638a5c2502333197019(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:20,087 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:20,087 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:20,090 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=13, startTime=1732532119871; duration=0sec 2024-11-25T10:55:20,090 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:20,091 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:20,091 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:20,093 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=13, startTime=1732532119852; duration=0sec 2024-11-25T10:55:20,094 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:20,094 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:20,100 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:20,100 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:20,101 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:20,101 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3a5bb642c84a44368ad414e8b177bf66, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6525a836ce3c462fb766406a53a6f927, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/ebb41d13ec6c46f58e52d90abe5716ad] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=35.2 K 2024-11-25T10:55:20,102 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a5bb642c84a44368ad414e8b177bf66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732532116879 2024-11-25T10:55:20,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741848_1024 (size=14341) 2024-11-25T10:55:20,105 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6525a836ce3c462fb766406a53a6f927, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732532117318 2024-11-25T10:55:20,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/986e4edbfb874e2ab467b243593051c9 2024-11-25T10:55:20,107 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ebb41d13ec6c46f58e52d90abe5716ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532118611 2024-11-25T10:55:20,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/29e8ba68d2344330a1aab25f8a332bdf is 50, key is test_row_0/B:col10/1732532119967/Put/seqid=0 2024-11-25T10:55:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532180157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,170 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:20,171 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/f4be4bc7ce584db695473f6830b118ee is 50, key is test_row_0/C:col10/1732532118720/Put/seqid=0 2024-11-25T10:55:20,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532180175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532180175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532180184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532180185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741849_1025 (size=12001) 2024-11-25T10:55:20,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/29e8ba68d2344330a1aab25f8a332bdf 2024-11-25T10:55:20,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741850_1026 (size=12104) 2024-11-25T10:55:20,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/be4ebe0e976145c2a31ec7af9e7b0d70 is 50, key is test_row_0/C:col10/1732532119967/Put/seqid=0 2024-11-25T10:55:20,314 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/f4be4bc7ce584db695473f6830b118ee as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/f4be4bc7ce584db695473f6830b118ee 2024-11-25T10:55:20,357 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into f4be4bc7ce584db695473f6830b118ee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:20,357 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:20,357 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=13, startTime=1732532119871; duration=0sec 2024-11-25T10:55:20,357 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:20,358 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:20,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741851_1027 (size=12001) 2024-11-25T10:55:20,369 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T10:55:20,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532180374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532180396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532180397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532180416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532180416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,479 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T10:55:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532180717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532180717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532180709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532180735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532180754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:20,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/be4ebe0e976145c2a31ec7af9e7b0d70 2024-11-25T10:55:20,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/986e4edbfb874e2ab467b243593051c9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/986e4edbfb874e2ab467b243593051c9 2024-11-25T10:55:20,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/986e4edbfb874e2ab467b243593051c9, entries=200, sequenceid=66, filesize=14.0 K 2024-11-25T10:55:20,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/29e8ba68d2344330a1aab25f8a332bdf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/29e8ba68d2344330a1aab25f8a332bdf 2024-11-25T10:55:20,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/29e8ba68d2344330a1aab25f8a332bdf, entries=150, sequenceid=66, filesize=11.7 K 2024-11-25T10:55:20,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/be4ebe0e976145c2a31ec7af9e7b0d70 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/be4ebe0e976145c2a31ec7af9e7b0d70 2024-11-25T10:55:20,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/be4ebe0e976145c2a31ec7af9e7b0d70, entries=150, sequenceid=66, filesize=11.7 K 2024-11-25T10:55:20,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 931ms, sequenceid=66, compaction requested=false 2024-11-25T10:55:20,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T10:55:21,018 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:21,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-25T10:55:21,023 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T10:55:21,041 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:21,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T10:55:21,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T10:55:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:21,205 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:55:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:21,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:21,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:21,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:21,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:21,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/c920a46673ad4c378de8891c505907e8 is 50, key is test_row_0/A:col10/1732532120032/Put/seqid=0 2024-11-25T10:55:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:21,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:21,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741852_1028 (size=12001) 2024-11-25T10:55:21,294 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/c920a46673ad4c378de8891c505907e8 2024-11-25T10:55:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532181308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532181314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T10:55:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/0ee8c33948c74eaabb312892eb04f63e is 50, key is test_row_0/B:col10/1732532120032/Put/seqid=0 2024-11-25T10:55:21,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532181341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532181350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532181378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741853_1029 (size=12001) 2024-11-25T10:55:21,414 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/0ee8c33948c74eaabb312892eb04f63e 2024-11-25T10:55:21,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/df64a57688e8492bba9d7e5d379eba5c is 50, key is test_row_0/C:col10/1732532120032/Put/seqid=0 2024-11-25T10:55:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532181458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532181464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532181489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532181490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532181517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741854_1030 (size=12001) 2024-11-25T10:55:21,545 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/df64a57688e8492bba9d7e5d379eba5c 2024-11-25T10:55:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/c920a46673ad4c378de8891c505907e8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/c920a46673ad4c378de8891c505907e8 2024-11-25T10:55:21,592 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/c920a46673ad4c378de8891c505907e8, entries=150, sequenceid=82, filesize=11.7 K 2024-11-25T10:55:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/0ee8c33948c74eaabb312892eb04f63e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0ee8c33948c74eaabb312892eb04f63e 2024-11-25T10:55:21,651 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0ee8c33948c74eaabb312892eb04f63e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-25T10:55:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/df64a57688e8492bba9d7e5d379eba5c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/df64a57688e8492bba9d7e5d379eba5c 2024-11-25T10:55:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T10:55:21,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,682 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/df64a57688e8492bba9d7e5d379eba5c, entries=150, sequenceid=82, filesize=11.7 K 2024-11-25T10:55:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532181677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,693 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 488ms, sequenceid=82, compaction requested=true 2024-11-25T10:55:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-25T10:55:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-25T10:55:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:21,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-25T10:55:21,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 668 msec 2024-11-25T10:55:21,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-25T10:55:21,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 708 msec 2024-11-25T10:55:21,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:21,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:21,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:21,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:21,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:21,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:21,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/3bc57fac3a224696bac99ed5fc14563c is 50, key is test_row_0/A:col10/1732532121717/Put/seqid=0 2024-11-25T10:55:21,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532181807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532181814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532181823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532181824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741855_1031 (size=12001) 2024-11-25T10:55:21,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532181937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532181941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532181967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532181946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:21,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532181995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532182164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T10:55:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532182175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532182176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,180 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:22,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-25T10:55:22,189 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T10:55:22,191 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:22,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:22,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532182189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/3bc57fac3a224696bac99ed5fc14563c 2024-11-25T10:55:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T10:55:22,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/1ff6629fe13c41bf9cb6eac035261b20 is 50, key is test_row_0/B:col10/1732532121717/Put/seqid=0 2024-11-25T10:55:22,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T10:55:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741856_1032 (size=12001) 2024-11-25T10:55:22,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/1ff6629fe13c41bf9cb6eac035261b20 2024-11-25T10:55:22,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/fa1b9b5f0a3b42d8a0005e6f051bd004 is 50, key is test_row_0/C:col10/1732532121717/Put/seqid=0 2024-11-25T10:55:22,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532182488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532182488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532182489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T10:55:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T10:55:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532182514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532182516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:22,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741857_1033 (size=12001) 2024-11-25T10:55:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:22,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T10:55:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T10:55:22,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T10:55:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:22,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:22,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:22,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/fa1b9b5f0a3b42d8a0005e6f051bd004 2024-11-25T10:55:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/3bc57fac3a224696bac99ed5fc14563c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3bc57fac3a224696bac99ed5fc14563c 2024-11-25T10:55:23,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532183026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532183034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:23,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T10:55:23,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:23,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:23,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:23,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:23,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:23,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3bc57fac3a224696bac99ed5fc14563c, entries=150, sequenceid=104, filesize=11.7 K 2024-11-25T10:55:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/1ff6629fe13c41bf9cb6eac035261b20 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1ff6629fe13c41bf9cb6eac035261b20 2024-11-25T10:55:23,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532183034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:23,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532183044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:23,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1ff6629fe13c41bf9cb6eac035261b20, entries=150, sequenceid=104, filesize=11.7 K 2024-11-25T10:55:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/fa1b9b5f0a3b42d8a0005e6f051bd004 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/fa1b9b5f0a3b42d8a0005e6f051bd004 2024-11-25T10:55:23,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/fa1b9b5f0a3b42d8a0005e6f051bd004, entries=150, sequenceid=104, filesize=11.7 K 2024-11-25T10:55:23,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1496ms, sequenceid=104, compaction requested=true 2024-11-25T10:55:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:23,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:23,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:23,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:55:23,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:23,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:55:23,224 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:55:23,225 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:55:23,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T10:55:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:23,239 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T10:55:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:23,239 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:55:23,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:23,240 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:23,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:23,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:23,240 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:23,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:23,240 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/f4be4bc7ce584db695473f6830b118ee, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/be4ebe0e976145c2a31ec7af9e7b0d70, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/df64a57688e8492bba9d7e5d379eba5c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/fa1b9b5f0a3b42d8a0005e6f051bd004] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=47.0 K 2024-11-25T10:55:23,240 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:55:23,241 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:23,241 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:23,241 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/8816887003784638a5c2502333197019, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/986e4edbfb874e2ab467b243593051c9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/c920a46673ad4c378de8891c505907e8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3bc57fac3a224696bac99ed5fc14563c] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=49.3 K 2024-11-25T10:55:23,242 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8816887003784638a5c2502333197019, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532118611 2024-11-25T10:55:23,242 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f4be4bc7ce584db695473f6830b118ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532118611 2024-11-25T10:55:23,243 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 986e4edbfb874e2ab467b243593051c9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732532119530 2024-11-25T10:55:23,244 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting be4ebe0e976145c2a31ec7af9e7b0d70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732532119567 2024-11-25T10:55:23,244 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c920a46673ad4c378de8891c505907e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732532120019 2024-11-25T10:55:23,245 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting df64a57688e8492bba9d7e5d379eba5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732532120019 2024-11-25T10:55:23,246 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1b9b5f0a3b42d8a0005e6f051bd004, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732532121339 2024-11-25T10:55:23,248 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc57fac3a224696bac99ed5fc14563c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732532121339 2024-11-25T10:55:23,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/de7991385e8f4f75a0aef956d8fa6b76 is 50, key is test_row_0/A:col10/1732532121806/Put/seqid=0 2024-11-25T10:55:23,325 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:23,327 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/0db9118d0080454d932a995c9d8e18f4 is 50, key is test_row_0/C:col10/1732532121717/Put/seqid=0 2024-11-25T10:55:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T10:55:23,346 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:23,347 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/7fcdfb01e7fc4ac3b948469116cab7f9 is 50, key is test_row_0/A:col10/1732532121717/Put/seqid=0 2024-11-25T10:55:23,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741858_1034 (size=12001) 2024-11-25T10:55:23,390 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/de7991385e8f4f75a0aef956d8fa6b76 2024-11-25T10:55:23,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741859_1035 (size=12241) 2024-11-25T10:55:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741860_1036 (size=12241) 2024-11-25T10:55:23,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/52feeba6d70c47f3aa13362c686fa7a6 is 50, key is test_row_0/B:col10/1732532121806/Put/seqid=0 2024-11-25T10:55:23,471 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/7fcdfb01e7fc4ac3b948469116cab7f9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/7fcdfb01e7fc4ac3b948469116cab7f9 2024-11-25T10:55:23,483 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/0db9118d0080454d932a995c9d8e18f4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/0db9118d0080454d932a995c9d8e18f4 2024-11-25T10:55:23,496 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 7fcdfb01e7fc4ac3b948469116cab7f9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:23,496 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:23,496 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=12, startTime=1732532123221; duration=0sec 2024-11-25T10:55:23,496 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:23,497 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:23,497 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:55:23,502 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 0db9118d0080454d932a995c9d8e18f4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:23,502 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:23,502 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=12, startTime=1732532123222; duration=0sec 2024-11-25T10:55:23,502 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:23,502 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:23,513 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:55:23,513 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:23,513 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:23,514 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ca3016a8e7643278c3bb1c82c05f318, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/29e8ba68d2344330a1aab25f8a332bdf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0ee8c33948c74eaabb312892eb04f63e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1ff6629fe13c41bf9cb6eac035261b20] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=47.0 K 2024-11-25T10:55:23,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741861_1037 (size=12001) 2024-11-25T10:55:23,515 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca3016a8e7643278c3bb1c82c05f318, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532118611 2024-11-25T10:55:23,516 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e8ba68d2344330a1aab25f8a332bdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732532119567 2024-11-25T10:55:23,517 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ee8c33948c74eaabb312892eb04f63e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732532120019 2024-11-25T10:55:23,517 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff6629fe13c41bf9cb6eac035261b20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732532121339 2024-11-25T10:55:23,521 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/52feeba6d70c47f3aa13362c686fa7a6 2024-11-25T10:55:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/8ae327c189c446658618d978a49af519 is 50, key is test_row_0/C:col10/1732532121806/Put/seqid=0 2024-11-25T10:55:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:23,589 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#26 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:23,590 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/65b0b0f6a44347ac9240182da8c413ee is 50, key is test_row_0/B:col10/1732532121717/Put/seqid=0 2024-11-25T10:55:23,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741862_1038 (size=12001) 2024-11-25T10:55:23,643 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/8ae327c189c446658618d978a49af519 2024-11-25T10:55:23,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741863_1039 (size=12241) 2024-11-25T10:55:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/de7991385e8f4f75a0aef956d8fa6b76 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/de7991385e8f4f75a0aef956d8fa6b76 2024-11-25T10:55:23,723 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/65b0b0f6a44347ac9240182da8c413ee as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/65b0b0f6a44347ac9240182da8c413ee 2024-11-25T10:55:23,730 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/de7991385e8f4f75a0aef956d8fa6b76, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T10:55:23,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/52feeba6d70c47f3aa13362c686fa7a6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/52feeba6d70c47f3aa13362c686fa7a6 2024-11-25T10:55:23,780 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 65b0b0f6a44347ac9240182da8c413ee(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:23,780 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:23,780 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=12, startTime=1732532123222; duration=0sec 2024-11-25T10:55:23,781 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:23,781 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:23,801 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/52feeba6d70c47f3aa13362c686fa7a6, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T10:55:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/8ae327c189c446658618d978a49af519 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/8ae327c189c446658618d978a49af519 2024-11-25T10:55:23,832 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/8ae327c189c446658618d978a49af519, entries=150, sequenceid=115, filesize=11.7 K 2024-11-25T10:55:23,850 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=67.09 KB/68700 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 612ms, sequenceid=115, compaction requested=false 2024-11-25T10:55:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-25T10:55:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-25T10:55:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:23,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:55:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:23,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T10:55:23,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6780 sec 2024-11-25T10:55:23,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/5f6c8749163d4f95b6e7e4f6a467d9dc is 50, key is test_row_0/A:col10/1732532123605/Put/seqid=0 2024-11-25T10:55:23,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7240 sec 2024-11-25T10:55:23,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741864_1040 (size=14441) 2024-11-25T10:55:23,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/5f6c8749163d4f95b6e7e4f6a467d9dc 2024-11-25T10:55:24,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/bc5f0cce115140f1b9869dc23ea7bc33 is 50, key is test_row_0/B:col10/1732532123605/Put/seqid=0 2024-11-25T10:55:24,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741865_1041 (size=12101) 2024-11-25T10:55:24,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532184194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532184197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532184242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532184251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532184254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T10:55:24,365 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:24,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532184364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532184364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-25T10:55:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532184364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T10:55:24,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532184365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,379 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,382 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:24,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:24,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532184381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T10:55:24,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/bc5f0cce115140f1b9869dc23ea7bc33 2024-11-25T10:55:24,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T10:55:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:24,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/a3b1e0dd54014a7d94a53e052e1ea9f0 is 50, key is test_row_0/C:col10/1732532123605/Put/seqid=0 2024-11-25T10:55:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532184577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532184578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532184585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741866_1042 (size=12101) 2024-11-25T10:55:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532184594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532184585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T10:55:24,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T10:55:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:24,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T10:55:24,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:24,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:24,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:24,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532184894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532184894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532184903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532184918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532184924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:24,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/a3b1e0dd54014a7d94a53e052e1ea9f0 2024-11-25T10:55:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T10:55:25,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/5f6c8749163d4f95b6e7e4f6a467d9dc as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5f6c8749163d4f95b6e7e4f6a467d9dc 2024-11-25T10:55:25,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T10:55:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:25,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:25,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5f6c8749163d4f95b6e7e4f6a467d9dc, entries=200, sequenceid=133, filesize=14.1 K 2024-11-25T10:55:25,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/bc5f0cce115140f1b9869dc23ea7bc33 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc5f0cce115140f1b9869dc23ea7bc33 2024-11-25T10:55:25,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc5f0cce115140f1b9869dc23ea7bc33, entries=150, sequenceid=133, filesize=11.8 K 2024-11-25T10:55:25,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/a3b1e0dd54014a7d94a53e052e1ea9f0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a3b1e0dd54014a7d94a53e052e1ea9f0 2024-11-25T10:55:25,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a3b1e0dd54014a7d94a53e052e1ea9f0, entries=150, sequenceid=133, filesize=11.8 K 2024-11-25T10:55:25,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1217ms, sequenceid=133, compaction requested=true 2024-11-25T10:55:25,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:25,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:25,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:25,090 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:25,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:55:25,092 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:25,092 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:25,092 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:25,092 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/7fcdfb01e7fc4ac3b948469116cab7f9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/de7991385e8f4f75a0aef956d8fa6b76, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5f6c8749163d4f95b6e7e4f6a467d9dc] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=37.8 K 2024-11-25T10:55:25,093 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fcdfb01e7fc4ac3b948469116cab7f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732532121339 2024-11-25T10:55:25,094 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting de7991385e8f4f75a0aef956d8fa6b76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732532121770 2024-11-25T10:55:25,094 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6c8749163d4f95b6e7e4f6a467d9dc, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732532123605 2024-11-25T10:55:25,096 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:25,099 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:25,099 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:25,099 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:25,099 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/65b0b0f6a44347ac9240182da8c413ee, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/52feeba6d70c47f3aa13362c686fa7a6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc5f0cce115140f1b9869dc23ea7bc33] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=35.5 K 2024-11-25T10:55:25,100 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b0b0f6a44347ac9240182da8c413ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732532121339 2024-11-25T10:55:25,113 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52feeba6d70c47f3aa13362c686fa7a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732532121770 2024-11-25T10:55:25,115 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc5f0cce115140f1b9869dc23ea7bc33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732532123605 2024-11-25T10:55:25,156 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:25,157 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/9b89b874be3b4684888fa63d4a8780ae is 50, key is test_row_0/A:col10/1732532123605/Put/seqid=0 2024-11-25T10:55:25,173 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:25,174 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/0cae9044b2f1496f8eafd1a93059bf83 is 50, key is test_row_0/B:col10/1732532123605/Put/seqid=0 2024-11-25T10:55:25,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T10:55:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:25,217 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:55:25,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:25,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:25,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:25,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741867_1043 (size=12443) 2024-11-25T10:55:25,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741868_1044 (size=12443) 2024-11-25T10:55:25,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/be3b9e43ea7944dd98e0907c64ea3b7b is 50, key is test_row_0/A:col10/1732532124187/Put/seqid=0 2024-11-25T10:55:25,331 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/9b89b874be3b4684888fa63d4a8780ae as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/9b89b874be3b4684888fa63d4a8780ae 2024-11-25T10:55:25,350 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/0cae9044b2f1496f8eafd1a93059bf83 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0cae9044b2f1496f8eafd1a93059bf83 2024-11-25T10:55:25,361 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 9b89b874be3b4684888fa63d4a8780ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:25,361 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:25,361 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=13, startTime=1732532125089; duration=0sec 2024-11-25T10:55:25,361 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:25,361 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:25,362 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:25,365 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:25,365 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:25,365 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:25,366 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/0db9118d0080454d932a995c9d8e18f4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/8ae327c189c446658618d978a49af519, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a3b1e0dd54014a7d94a53e052e1ea9f0] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=35.5 K 2024-11-25T10:55:25,367 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db9118d0080454d932a995c9d8e18f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732532121339 2024-11-25T10:55:25,368 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae327c189c446658618d978a49af519, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732532121770 2024-11-25T10:55:25,370 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b1e0dd54014a7d94a53e052e1ea9f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732532123605 2024-11-25T10:55:25,372 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 0cae9044b2f1496f8eafd1a93059bf83(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:25,372 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:25,372 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=13, startTime=1732532125090; duration=0sec 2024-11-25T10:55:25,372 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:25,373 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:25,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741869_1045 (size=12151) 2024-11-25T10:55:25,400 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#33 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:25,401 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/d23f82ab2cae46ae8c3751ad2efd0294 is 50, key is test_row_0/C:col10/1732532123605/Put/seqid=0 2024-11-25T10:55:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:25,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741870_1046 (size=12443) 2024-11-25T10:55:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T10:55:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532185532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532185538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532185546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532185549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532185557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532185675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532185675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532185680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532185683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532185695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,795 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/be3b9e43ea7944dd98e0907c64ea3b7b 2024-11-25T10:55:25,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/ba5ce059ee594efea2f6c3e6094e395b is 50, key is test_row_0/B:col10/1732532124187/Put/seqid=0 2024-11-25T10:55:25,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741871_1047 (size=12151) 2024-11-25T10:55:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532185885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532185886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532185893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532185904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:25,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532185913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:25,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,001 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/d23f82ab2cae46ae8c3751ad2efd0294 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d23f82ab2cae46ae8c3751ad2efd0294 2024-11-25T10:55:26,045 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into d23f82ab2cae46ae8c3751ad2efd0294(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:26,045 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:26,045 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=13, startTime=1732532125090; duration=0sec 2024-11-25T10:55:26,045 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:26,045 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:26,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532186194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532186194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532186214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532186236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532186237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,293 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/ba5ce059ee594efea2f6c3e6094e395b 2024-11-25T10:55:26,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/9526ed94c77349208c527837805b4dde is 50, key is test_row_0/C:col10/1732532124187/Put/seqid=0 2024-11-25T10:55:26,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741872_1048 (size=12151) 2024-11-25T10:55:26,347 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/9526ed94c77349208c527837805b4dde 2024-11-25T10:55:26,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/be3b9e43ea7944dd98e0907c64ea3b7b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/be3b9e43ea7944dd98e0907c64ea3b7b 2024-11-25T10:55:26,375 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/be3b9e43ea7944dd98e0907c64ea3b7b, entries=150, sequenceid=150, filesize=11.9 K 2024-11-25T10:55:26,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/ba5ce059ee594efea2f6c3e6094e395b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/ba5ce059ee594efea2f6c3e6094e395b 2024-11-25T10:55:26,390 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/ba5ce059ee594efea2f6c3e6094e395b, entries=150, sequenceid=150, filesize=11.9 K 2024-11-25T10:55:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/9526ed94c77349208c527837805b4dde as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/9526ed94c77349208c527837805b4dde 2024-11-25T10:55:26,407 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/9526ed94c77349208c527837805b4dde, entries=150, sequenceid=150, filesize=11.9 K 2024-11-25T10:55:26,409 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1191ms, sequenceid=150, compaction requested=false 2024-11-25T10:55:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-25T10:55:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-25T10:55:26,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T10:55:26,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0290 sec 2024-11-25T10:55:26,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0490 sec 2024-11-25T10:55:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T10:55:26,535 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:26,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-25T10:55:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T10:55:26,551 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:26,556 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:26,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T10:55:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:26,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:55:26,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:26,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:26,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:26,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:26,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:26,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:26,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:26,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/d3676e01b11e479ca093e94beb5b4c59 is 50, key is test_row_0/A:col10/1732532125543/Put/seqid=0 2024-11-25T10:55:26,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741873_1049 (size=12151) 2024-11-25T10:55:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532186795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532186796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532186798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532186802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532186804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T10:55:26,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:26,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:26,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532186917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532186918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532186921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532186939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532186944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:27,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532187141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532187144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532187144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532187164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532187166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T10:55:27,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/d3676e01b11e479ca093e94beb5b4c59 2024-11-25T10:55:27,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:27,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:27,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/805aab0cd6cb40199291666511abb4f9 is 50, key is test_row_0/B:col10/1732532125543/Put/seqid=0 2024-11-25T10:55:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741874_1050 (size=12151) 2024-11-25T10:55:27,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532187459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532187465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532187465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532187487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532187488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:27,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T10:55:27,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:27,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/805aab0cd6cb40199291666511abb4f9 2024-11-25T10:55:27,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/b5d4c2c54b5f44e88e3909a2d2a2d482 is 50, key is test_row_0/C:col10/1732532125543/Put/seqid=0 2024-11-25T10:55:27,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741875_1051 (size=12151) 2024-11-25T10:55:27,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/b5d4c2c54b5f44e88e3909a2d2a2d482 2024-11-25T10:55:27,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/d3676e01b11e479ca093e94beb5b4c59 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d3676e01b11e479ca093e94beb5b4c59 2024-11-25T10:55:27,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d3676e01b11e479ca093e94beb5b4c59, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T10:55:27,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/805aab0cd6cb40199291666511abb4f9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/805aab0cd6cb40199291666511abb4f9 2024-11-25T10:55:27,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/805aab0cd6cb40199291666511abb4f9, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T10:55:27,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/b5d4c2c54b5f44e88e3909a2d2a2d482 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b5d4c2c54b5f44e88e3909a2d2a2d482 2024-11-25T10:55:27,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b5d4c2c54b5f44e88e3909a2d2a2d482, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T10:55:27,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1146ms, sequenceid=172, compaction requested=true 2024-11-25T10:55:27,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:27,864 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:27,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:27,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:27,864 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:27,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:27,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:27,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:27,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:27,868 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:27,868 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:27,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T10:55:27,868 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,868 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/9b89b874be3b4684888fa63d4a8780ae, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/be3b9e43ea7944dd98e0907c64ea3b7b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d3676e01b11e479ca093e94beb5b4c59] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=35.9 K 2024-11-25T10:55:27,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,869 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:55:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:27,870 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:27,870 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b89b874be3b4684888fa63d4a8780ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732532123605 2024-11-25T10:55:27,870 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:27,870 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:27,870 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0cae9044b2f1496f8eafd1a93059bf83, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/ba5ce059ee594efea2f6c3e6094e395b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/805aab0cd6cb40199291666511abb4f9] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=35.9 K 2024-11-25T10:55:27,871 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting be3b9e43ea7944dd98e0907c64ea3b7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732532124187 2024-11-25T10:55:27,871 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cae9044b2f1496f8eafd1a93059bf83, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732532123605 2024-11-25T10:55:27,871 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ba5ce059ee594efea2f6c3e6094e395b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732532124187 2024-11-25T10:55:27,872 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3676e01b11e479ca093e94beb5b4c59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532125498 2024-11-25T10:55:27,872 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 805aab0cd6cb40199291666511abb4f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532125498 2024-11-25T10:55:27,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d is 50, key is test_row_0/A:col10/1732532126788/Put/seqid=0 2024-11-25T10:55:27,946 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#40 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:27,947 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/46d98ce98dd04f82a85dc339c4dac9d1 is 50, key is test_row_0/A:col10/1732532125543/Put/seqid=0 2024-11-25T10:55:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741876_1052 (size=12151) 2024-11-25T10:55:27,950 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:27,951 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/38a5fb13aad34c5e9944b46155882b4f is 50, key is test_row_0/B:col10/1732532125543/Put/seqid=0 2024-11-25T10:55:27,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:27,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741877_1053 (size=12595) 2024-11-25T10:55:28,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741878_1054 (size=12595) 2024-11-25T10:55:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532188034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532188035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532188048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532188048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532188049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,051 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/38a5fb13aad34c5e9944b46155882b4f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/38a5fb13aad34c5e9944b46155882b4f 2024-11-25T10:55:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,081 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 38a5fb13aad34c5e9944b46155882b4f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:28,081 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:28,081 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=13, startTime=1732532127864; duration=0sec 2024-11-25T10:55:28,081 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:28,081 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:28,081 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:28,085 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:28,085 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:28,085 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:28,085 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d23f82ab2cae46ae8c3751ad2efd0294, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/9526ed94c77349208c527837805b4dde, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b5d4c2c54b5f44e88e3909a2d2a2d482] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=35.9 K 2024-11-25T10:55:28,086 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d23f82ab2cae46ae8c3751ad2efd0294, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732532123605 2024-11-25T10:55:28,086 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9526ed94c77349208c527837805b4dde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732532124187 2024-11-25T10:55:28,087 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d4c2c54b5f44e88e3909a2d2a2d482, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532125498 2024-11-25T10:55:28,107 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:28,107 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/7740132fc6e849a99661876fdbf83db9 is 50, key is test_row_0/C:col10/1732532125543/Put/seqid=0 2024-11-25T10:55:28,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741879_1055 (size=12595) 2024-11-25T10:55:28,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532188155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532188155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532188155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532188156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532188156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,350 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d 2024-11-25T10:55:28,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532188364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532188364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532188364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532188365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532188365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/814bd5a0db37438bb3aa1ee073228587 is 50, key is test_row_0/B:col10/1732532126788/Put/seqid=0 2024-11-25T10:55:28,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741880_1056 (size=12151) 2024-11-25T10:55:28,427 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/46d98ce98dd04f82a85dc339c4dac9d1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/46d98ce98dd04f82a85dc339c4dac9d1 2024-11-25T10:55:28,453 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 46d98ce98dd04f82a85dc339c4dac9d1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:28,454 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:28,454 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=13, startTime=1732532127863; duration=0sec 2024-11-25T10:55:28,454 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:28,454 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:28,557 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/7740132fc6e849a99661876fdbf83db9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/7740132fc6e849a99661876fdbf83db9 2024-11-25T10:55:28,572 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 7740132fc6e849a99661876fdbf83db9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:28,572 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:28,573 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=13, startTime=1732532127864; duration=0sec 2024-11-25T10:55:28,573 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:28,573 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:28,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532188674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532188674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532188689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532188690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T10:55:28,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532188690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:28,828 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/814bd5a0db37438bb3aa1ee073228587 2024-11-25T10:55:28,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/cd5d5aedd6a64cff823569a5accf4ef6 is 50, key is test_row_0/C:col10/1732532126788/Put/seqid=0 2024-11-25T10:55:28,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741881_1057 (size=12151) 2024-11-25T10:55:28,968 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/cd5d5aedd6a64cff823569a5accf4ef6 2024-11-25T10:55:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d 2024-11-25T10:55:29,047 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d, entries=150, sequenceid=187, filesize=11.9 K 2024-11-25T10:55:29,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/814bd5a0db37438bb3aa1ee073228587 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/814bd5a0db37438bb3aa1ee073228587 2024-11-25T10:55:29,139 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/814bd5a0db37438bb3aa1ee073228587, entries=150, sequenceid=187, filesize=11.9 K 2024-11-25T10:55:29,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/cd5d5aedd6a64cff823569a5accf4ef6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd5d5aedd6a64cff823569a5accf4ef6 2024-11-25T10:55:29,169 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd5d5aedd6a64cff823569a5accf4ef6, entries=150, sequenceid=187, filesize=11.9 K 2024-11-25T10:55:29,175 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1307ms, sequenceid=187, compaction requested=false 2024-11-25T10:55:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-25T10:55:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-25T10:55:29,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-25T10:55:29,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6230 sec 2024-11-25T10:55:29,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.6480 sec 2024-11-25T10:55:29,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:55:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:29,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/e40700c489974f36b051768e86cc4517 is 50, key is test_row_0/A:col10/1732532128034/Put/seqid=0 2024-11-25T10:55:29,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532189242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532189250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532189259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741882_1058 (size=14541) 2024-11-25T10:55:29,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/e40700c489974f36b051768e86cc4517 2024-11-25T10:55:29,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532189274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532189295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/66815f9fd2ae4b08a2df30becbefb86b is 50, key is test_row_0/B:col10/1732532128034/Put/seqid=0 2024-11-25T10:55:29,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532189373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741883_1059 (size=12151) 2024-11-25T10:55:29,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/66815f9fd2ae4b08a2df30becbefb86b 2024-11-25T10:55:29,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532189384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532189404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532189409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/18f10eb0c1694138af8e96d451c41a55 is 50, key is test_row_0/C:col10/1732532128034/Put/seqid=0 2024-11-25T10:55:29,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532189425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741884_1060 (size=12151) 2024-11-25T10:55:29,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/18f10eb0c1694138af8e96d451c41a55 2024-11-25T10:55:29,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/e40700c489974f36b051768e86cc4517 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/e40700c489974f36b051768e86cc4517 2024-11-25T10:55:29,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/e40700c489974f36b051768e86cc4517, entries=200, sequenceid=208, filesize=14.2 K 2024-11-25T10:55:29,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/66815f9fd2ae4b08a2df30becbefb86b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/66815f9fd2ae4b08a2df30becbefb86b 2024-11-25T10:55:29,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/66815f9fd2ae4b08a2df30becbefb86b, entries=150, sequenceid=208, filesize=11.9 K 2024-11-25T10:55:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/18f10eb0c1694138af8e96d451c41a55 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/18f10eb0c1694138af8e96d451c41a55 2024-11-25T10:55:29,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/18f10eb0c1694138af8e96d451c41a55, entries=150, sequenceid=208, filesize=11.9 K 2024-11-25T10:55:29,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 320ms, sequenceid=208, compaction requested=true 2024-11-25T10:55:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:29,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:55:29,521 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:29,521 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:29,523 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:29,523 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:29,523 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:29,523 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:29,523 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:29,523 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/38a5fb13aad34c5e9944b46155882b4f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/814bd5a0db37438bb3aa1ee073228587, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/66815f9fd2ae4b08a2df30becbefb86b] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.0 K 2024-11-25T10:55:29,523 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:29,524 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/46d98ce98dd04f82a85dc339c4dac9d1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/e40700c489974f36b051768e86cc4517] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=38.4 K 2024-11-25T10:55:29,524 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d98ce98dd04f82a85dc339c4dac9d1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532125498 2024-11-25T10:55:29,524 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a5fb13aad34c5e9944b46155882b4f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532125498 2024-11-25T10:55:29,525 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a7d3a1ee94b54cf6ad6dc8006d63fb2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732532126779 2024-11-25T10:55:29,525 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 814bd5a0db37438bb3aa1ee073228587, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732532126779 2024-11-25T10:55:29,525 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e40700c489974f36b051768e86cc4517, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732532128002 2024-11-25T10:55:29,526 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66815f9fd2ae4b08a2df30becbefb86b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732532128034 2024-11-25T10:55:29,543 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:29,544 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/6c9bbe677e6443468556a6c687851e41 is 50, key is test_row_0/A:col10/1732532128034/Put/seqid=0 2024-11-25T10:55:29,547 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:29,548 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/05ed2d7cb17d40c481169583538dabd9 is 50, key is test_row_0/B:col10/1732532128034/Put/seqid=0 2024-11-25T10:55:29,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741886_1062 (size=12697) 2024-11-25T10:55:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:29,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T10:55:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:29,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:29,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741885_1061 (size=12697) 2024-11-25T10:55:29,626 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/05ed2d7cb17d40c481169583538dabd9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/05ed2d7cb17d40c481169583538dabd9 2024-11-25T10:55:29,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/d4dd8df0936746d5a3549fec8ccb7524 is 50, key is test_row_0/A:col10/1732532129250/Put/seqid=0 2024-11-25T10:55:29,641 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/6c9bbe677e6443468556a6c687851e41 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6c9bbe677e6443468556a6c687851e41 2024-11-25T10:55:29,643 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 05ed2d7cb17d40c481169583538dabd9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:29,643 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:29,643 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=13, startTime=1732532129520; duration=0sec 2024-11-25T10:55:29,643 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:29,644 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:29,644 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:29,653 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:29,653 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:29,653 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:29,653 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/7740132fc6e849a99661876fdbf83db9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd5d5aedd6a64cff823569a5accf4ef6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/18f10eb0c1694138af8e96d451c41a55] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.0 K 2024-11-25T10:55:29,663 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7740132fc6e849a99661876fdbf83db9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532125498 2024-11-25T10:55:29,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532189653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,678 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd5d5aedd6a64cff823569a5accf4ef6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732532126779 2024-11-25T10:55:29,684 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18f10eb0c1694138af8e96d451c41a55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732532128034 2024-11-25T10:55:29,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741887_1063 (size=14541) 2024-11-25T10:55:29,695 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 6c9bbe677e6443468556a6c687851e41(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:29,695 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:29,695 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=13, startTime=1732532129520; duration=0sec 2024-11-25T10:55:29,695 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:29,695 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:29,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/d4dd8df0936746d5a3549fec8ccb7524 2024-11-25T10:55:29,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532189673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532189675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532189675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532189680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,759 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:29,769 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/37e8e4af8e1e4a7384acc15735a2db5f is 50, key is test_row_0/C:col10/1732532128034/Put/seqid=0 2024-11-25T10:55:29,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7ea063909ae246da961f2b654805f986 is 50, key is test_row_0/B:col10/1732532129250/Put/seqid=0 2024-11-25T10:55:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532189785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532189813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532189805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532189813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532189824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741888_1064 (size=12697) 2024-11-25T10:55:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:29,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741889_1065 (size=12151) 2024-11-25T10:55:29,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7ea063909ae246da961f2b654805f986 2024-11-25T10:55:29,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3ca9c3c15893489d9b14e847a902c82c is 50, key is test_row_0/C:col10/1732532129250/Put/seqid=0 2024-11-25T10:55:29,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741890_1066 (size=12151) 2024-11-25T10:55:29,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3ca9c3c15893489d9b14e847a902c82c 2024-11-25T10:55:29,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/d4dd8df0936746d5a3549fec8ccb7524 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d4dd8df0936746d5a3549fec8ccb7524 2024-11-25T10:55:29,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d4dd8df0936746d5a3549fec8ccb7524, entries=200, sequenceid=224, filesize=14.2 K 2024-11-25T10:55:29,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7ea063909ae246da961f2b654805f986 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ea063909ae246da961f2b654805f986 2024-11-25T10:55:29,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ea063909ae246da961f2b654805f986, entries=150, sequenceid=224, filesize=11.9 K 2024-11-25T10:55:29,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3ca9c3c15893489d9b14e847a902c82c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3ca9c3c15893489d9b14e847a902c82c 2024-11-25T10:55:29,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532189994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3ca9c3c15893489d9b14e847a902c82c, entries=150, sequenceid=224, filesize=11.9 K 2024-11-25T10:55:30,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 426ms, sequenceid=224, compaction requested=false 2024-11-25T10:55:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:30,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:55:30,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:30,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:30,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:30,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:30,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:30,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:30,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/444d8b898a6d4cdc9b003990480bfc68 is 50, key is test_row_0/A:col10/1732532129648/Put/seqid=0 2024-11-25T10:55:30,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741891_1067 (size=12151) 2024-11-25T10:55:30,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/444d8b898a6d4cdc9b003990480bfc68 2024-11-25T10:55:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532190072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532190079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532190092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532190094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7e8764d217294ec98f4bc88c68e1fff2 is 50, key is test_row_0/B:col10/1732532129648/Put/seqid=0 2024-11-25T10:55:30,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741892_1068 (size=12151) 2024-11-25T10:55:30,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7e8764d217294ec98f4bc88c68e1fff2 2024-11-25T10:55:30,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532190198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532190204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532190204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/d72aa8f7132941bb92a6f66ada6fa41e is 50, key is test_row_0/C:col10/1732532129648/Put/seqid=0 2024-11-25T10:55:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532190214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,310 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/37e8e4af8e1e4a7384acc15735a2db5f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/37e8e4af8e1e4a7384acc15735a2db5f 2024-11-25T10:55:30,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741893_1069 (size=12151) 2024-11-25T10:55:30,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532190308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,362 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 37e8e4af8e1e4a7384acc15735a2db5f(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:30,363 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:30,363 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=13, startTime=1732532129520; duration=0sec 2024-11-25T10:55:30,363 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:30,363 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:30,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532190413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532190424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532190425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532190425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/d72aa8f7132941bb92a6f66ada6fa41e 2024-11-25T10:55:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T10:55:30,724 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:30,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532190734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-25T10:55:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T10:55:30,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:30,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:30,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532190737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/444d8b898a6d4cdc9b003990480bfc68 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/444d8b898a6d4cdc9b003990480bfc68 2024-11-25T10:55:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532190744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532190745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/444d8b898a6d4cdc9b003990480bfc68, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T10:55:30,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7e8764d217294ec98f4bc88c68e1fff2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7e8764d217294ec98f4bc88c68e1fff2 2024-11-25T10:55:30,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7e8764d217294ec98f4bc88c68e1fff2, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T10:55:30,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/d72aa8f7132941bb92a6f66ada6fa41e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d72aa8f7132941bb92a6f66ada6fa41e 2024-11-25T10:55:30,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532190824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:30,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d72aa8f7132941bb92a6f66ada6fa41e, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T10:55:30,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 815ms, sequenceid=245, compaction requested=true 2024-11-25T10:55:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:30,848 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:30,849 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:30,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:30,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T10:55:30,862 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:30,862 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:30,862 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:30,862 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/05ed2d7cb17d40c481169583538dabd9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ea063909ae246da961f2b654805f986, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7e8764d217294ec98f4bc88c68e1fff2] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.1 K 2024-11-25T10:55:30,863 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ed2d7cb17d40c481169583538dabd9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732532128034 2024-11-25T10:55:30,863 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ea063909ae246da961f2b654805f986, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532129247 2024-11-25T10:55:30,864 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e8764d217294ec98f4bc88c68e1fff2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532129648 2024-11-25T10:55:30,864 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:30,864 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:30,865 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:30,865 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6c9bbe677e6443468556a6c687851e41, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d4dd8df0936746d5a3549fec8ccb7524, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/444d8b898a6d4cdc9b003990480bfc68] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=38.5 K 2024-11-25T10:55:30,872 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9bbe677e6443468556a6c687851e41, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732532128034 2024-11-25T10:55:30,876 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4dd8df0936746d5a3549fec8ccb7524, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532129247 2024-11-25T10:55:30,878 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 444d8b898a6d4cdc9b003990480bfc68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532129648 2024-11-25T10:55:30,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T10:55:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:30,902 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:55:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:30,930 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:30,931 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/bc1f426644314e0381c41ae0cb77cc38 is 50, key is test_row_0/B:col10/1732532129648/Put/seqid=0 2024-11-25T10:55:30,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/553e054a6e3a449b8530cbcc721bdf6e is 50, key is test_row_0/A:col10/1732532130071/Put/seqid=0 2024-11-25T10:55:30,970 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:30,970 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/5c48aa020ea941e4ac8dfe4d57c68fd9 is 50, key is test_row_0/A:col10/1732532129648/Put/seqid=0 2024-11-25T10:55:30,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741894_1070 (size=12201) 2024-11-25T10:55:31,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741895_1071 (size=12799) 2024-11-25T10:55:31,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741896_1072 (size=12799) 2024-11-25T10:55:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T10:55:31,067 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/bc1f426644314e0381c41ae0cb77cc38 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc1f426644314e0381c41ae0cb77cc38 2024-11-25T10:55:31,090 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/5c48aa020ea941e4ac8dfe4d57c68fd9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5c48aa020ea941e4ac8dfe4d57c68fd9 2024-11-25T10:55:31,113 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into bc1f426644314e0381c41ae0cb77cc38(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:31,113 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:31,113 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=13, startTime=1732532130848; duration=0sec 2024-11-25T10:55:31,113 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:31,113 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:31,113 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:31,115 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:31,115 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:31,115 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:31,115 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/37e8e4af8e1e4a7384acc15735a2db5f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3ca9c3c15893489d9b14e847a902c82c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d72aa8f7132941bb92a6f66ada6fa41e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.1 K 2024-11-25T10:55:31,122 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 37e8e4af8e1e4a7384acc15735a2db5f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732532128034 2024-11-25T10:55:31,122 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 5c48aa020ea941e4ac8dfe4d57c68fd9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:31,122 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:31,122 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=13, startTime=1732532130848; duration=0sec 2024-11-25T10:55:31,122 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:31,122 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:31,125 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca9c3c15893489d9b14e847a902c82c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532129247 2024-11-25T10:55:31,127 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d72aa8f7132941bb92a6f66ada6fa41e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532129648 2024-11-25T10:55:31,238 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#60 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:31,239 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6f26bfa60706407db833d276586b985e is 50, key is test_row_0/C:col10/1732532129648/Put/seqid=0 2024-11-25T10:55:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:31,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741897_1073 (size=12799) 2024-11-25T10:55:31,333 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6f26bfa60706407db833d276586b985e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6f26bfa60706407db833d276586b985e 2024-11-25T10:55:31,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532191344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532191347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532191350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532191358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,369 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 6f26bfa60706407db833d276586b985e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:31,369 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:31,369 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=13, startTime=1732532130849; duration=0sec 2024-11-25T10:55:31,369 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:31,369 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:31,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T10:55:31,401 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/553e054a6e3a449b8530cbcc721bdf6e 2024-11-25T10:55:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7fee6b2ab12d45339319c5522b26e5c6 is 50, key is test_row_0/B:col10/1732532130071/Put/seqid=0 2024-11-25T10:55:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532191464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532191465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532191469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532191475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741898_1074 (size=12201) 2024-11-25T10:55:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532191688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532191689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532191689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532191701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532191844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:31,852 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in 619a47d3e2a0b4c567d3fd0e3f1893d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:55:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:31,896 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7fee6b2ab12d45339319c5522b26e5c6 2024-11-25T10:55:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T10:55:31,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/a1c9071575d14af8bf915287833809bc is 50, key is test_row_0/C:col10/1732532130071/Put/seqid=0 2024-11-25T10:55:31,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741899_1075 (size=12201) 2024-11-25T10:55:31,998 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/a1c9071575d14af8bf915287833809bc 2024-11-25T10:55:32,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532192005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:32,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532192006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532192014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:32,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/553e054a6e3a449b8530cbcc721bdf6e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/553e054a6e3a449b8530cbcc721bdf6e 2024-11-25T10:55:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532192014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:32,058 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/553e054a6e3a449b8530cbcc721bdf6e, entries=150, sequenceid=259, filesize=11.9 K 2024-11-25T10:55:32,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7fee6b2ab12d45339319c5522b26e5c6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7fee6b2ab12d45339319c5522b26e5c6 2024-11-25T10:55:32,068 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7fee6b2ab12d45339319c5522b26e5c6, entries=150, sequenceid=259, filesize=11.9 K 2024-11-25T10:55:32,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/a1c9071575d14af8bf915287833809bc as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a1c9071575d14af8bf915287833809bc 2024-11-25T10:55:32,077 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a1c9071575d14af8bf915287833809bc, entries=150, sequenceid=259, filesize=11.9 K 2024-11-25T10:55:32,089 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1186ms, sequenceid=259, compaction requested=false 2024-11-25T10:55:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:32,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-25T10:55:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-25T10:55:32,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T10:55:32,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3540 sec 2024-11-25T10:55:32,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3720 sec 2024-11-25T10:55:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:32,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-25T10:55:32,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:32,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532131343, max timestamp=1732532132522Num uniques 859; 2024-11-25T10:55:32,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532131343, max timestamp=1732532132522Num uniques 859; 2024-11-25T10:55:32,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:32,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:32,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:55:32,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:55:32,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732532131343, max timestamp=1732532132522Num uniques 859; 2024-11-25T10:55:32,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:32,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:32,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T10:55:32,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:32,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/cebfe3e5878a447fa5ae556ab4b82bf8 is 50, key is test_row_0/A:col10/1732532132522/Put/seqid=0 2024-11-25T10:55:32,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741900_1076 (size=12301) 2024-11-25T10:55:32,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532132597, max timestamp=1732532132637Num uniques 859; 2024-11-25T10:55:32,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:55:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732532132597, max timestamp=1732532132637Num uniques 859; 2024-11-25T10:55:32,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T10:55:32,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732532132597, max timestamp=1732532132642Num uniques 859; 2024-11-25T10:55:32,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T10:55:32,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732532132695, max timestamp=1732532132781Num uniques 1068; 2024-11-25T10:55:32,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-25T10:55:32,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732532132695, max timestamp=1732532132781Num uniques 1009; 2024-11-25T10:55:32,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-25T10:55:32,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732532132695, max timestamp=1732532132781Num uniques 1009; 2024-11-25T10:55:32,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-25T10:55:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T10:55:32,905 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:32,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-25T10:55:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:32,933 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:32,941 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:32,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:32,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732532132871, max timestamp=1732532132932Num uniques 1027; 2024-11-25T10:55:32,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-11-25T10:55:32,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732532132871, max timestamp=1732532132932Num uniques 1027; 2024-11-25T10:55:32,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-11-25T10:55:32,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732532132871, max timestamp=1732532132932Num uniques 1027; 2024-11-25T10:55:32,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=17262, suffix heap size=134776,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=377 2024-11-25T10:55:32,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/cebfe3e5878a447fa5ae556ab4b82bf8 2024-11-25T10:55:33,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/1d797c3a9e7740729812c16943e96c7f is 50, key is test_row_0/B:col10/1732532132522/Put/seqid=0 2024-11-25T10:55:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:33,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732532132985, max timestamp=1732532133027Num uniques 1236; 2024-11-25T10:55:33,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732532132985, max timestamp=1732532133027Num uniques 1236; 2024-11-25T10:55:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-25T10:55:33,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-25T10:55:33,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732532132985, max timestamp=1732532133027Num uniques 1236; 2024-11-25T10:55:33,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=8516, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-25T10:55:33,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741901_1077 (size=12301) 2024-11-25T10:55:33,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/1d797c3a9e7740729812c16943e96c7f 2024-11-25T10:55:33,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/cd4ea06198b4456892165ca83a51eb0d is 50, key is test_row_0/C:col10/1732532132522/Put/seqid=0 2024-11-25T10:55:33,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741902_1078 (size=12301) 2024-11-25T10:55:33,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/cd4ea06198b4456892165ca83a51eb0d 2024-11-25T10:55:33,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732532133070, max timestamp=1732532133146Num uniques 1045; 2024-11-25T10:55:33,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-25T10:55:33,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/cebfe3e5878a447fa5ae556ab4b82bf8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/cebfe3e5878a447fa5ae556ab4b82bf8 2024-11-25T10:55:33,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732532133070, max timestamp=1732532133146Num uniques 1045; 2024-11-25T10:55:33,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-25T10:55:33,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732532133070, max timestamp=1732532133146Num uniques 1045; 2024-11-25T10:55:33,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-25T10:55:33,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/cebfe3e5878a447fa5ae556ab4b82bf8, entries=150, sequenceid=283, filesize=12.0 K 2024-11-25T10:55:33,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/1d797c3a9e7740729812c16943e96c7f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1d797c3a9e7740729812c16943e96c7f 2024-11-25T10:55:33,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1d797c3a9e7740729812c16943e96c7f, entries=150, sequenceid=283, filesize=12.0 K 2024-11-25T10:55:33,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/cd4ea06198b4456892165ca83a51eb0d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd4ea06198b4456892165ca83a51eb0d 2024-11-25T10:55:33,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd4ea06198b4456892165ca83a51eb0d, entries=150, sequenceid=283, filesize=12.0 K 2024-11-25T10:55:33,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.44 KB/27075, heapSize ~65.27 KB/66840, currentSize=92.14 KB/94350 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 694ms, sequenceid=283, compaction requested=true 2024-11-25T10:55:33,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:33,223 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:33,224 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:33,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=105.56 KB heapSize=297.86 KB 2024-11-25T10:55:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:33,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:33,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:33,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:33,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:33,226 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:33,227 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:33,227 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,227 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:33,227 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5c48aa020ea941e4ac8dfe4d57c68fd9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/553e054a6e3a449b8530cbcc721bdf6e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/cebfe3e5878a447fa5ae556ab4b82bf8] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.4 K 2024-11-25T10:55:33,228 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:33,229 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,233 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc1f426644314e0381c41ae0cb77cc38, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7fee6b2ab12d45339319c5522b26e5c6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1d797c3a9e7740729812c16943e96c7f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.4 K 2024-11-25T10:55:33,233 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c48aa020ea941e4ac8dfe4d57c68fd9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532129648 2024-11-25T10:55:33,234 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1f426644314e0381c41ae0cb77cc38, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532129648 2024-11-25T10:55:33,234 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553e054a6e3a449b8530cbcc721bdf6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732532130071 2024-11-25T10:55:33,236 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fee6b2ab12d45339319c5522b26e5c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732532130071 2024-11-25T10:55:33,236 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cebfe3e5878a447fa5ae556ab4b82bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732532131343 2024-11-25T10:55:33,238 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d797c3a9e7740729812c16943e96c7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732532131343 2024-11-25T10:55:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:33,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/6600c64731aa45ad943a566a05ed5405 is 50, key is test_row_0/A:col10/1732532133218/Put/seqid=0 2024-11-25T10:55:33,287 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:33,288 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a9ac6547f03a467bac65e906b1d0f7f7 is 50, key is test_row_0/A:col10/1732532132522/Put/seqid=0 2024-11-25T10:55:33,291 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:33,292 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7bf99ea59c334404b635b9bb2651e027 is 50, key is test_row_0/B:col10/1732532132522/Put/seqid=0 2024-11-25T10:55:33,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532193296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532193296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532193317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532193320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741904_1080 (size=13051) 2024-11-25T10:55:33,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741903_1079 (size=12301) 2024-11-25T10:55:33,408 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/7bf99ea59c334404b635b9bb2651e027 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7bf99ea59c334404b635b9bb2651e027 2024-11-25T10:55:33,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741905_1081 (size=13051) 2024-11-25T10:55:33,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532193429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532193430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532193444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,449 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 7bf99ea59c334404b635b9bb2651e027(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:33,450 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:33,450 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=13, startTime=1732532133224; duration=0sec 2024-11-25T10:55:33,450 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:33,450 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:33,450 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:33,456 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a9ac6547f03a467bac65e906b1d0f7f7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9ac6547f03a467bac65e906b1d0f7f7 2024-11-25T10:55:33,465 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:33,465 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:33,465 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,466 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6f26bfa60706407db833d276586b985e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a1c9071575d14af8bf915287833809bc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd4ea06198b4456892165ca83a51eb0d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.4 K 2024-11-25T10:55:33,469 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f26bfa60706407db833d276586b985e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532129648 2024-11-25T10:55:33,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532193454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,480 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c9071575d14af8bf915287833809bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732532130071 2024-11-25T10:55:33,481 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4ea06198b4456892165ca83a51eb0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732532131343 2024-11-25T10:55:33,500 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into a9ac6547f03a467bac65e906b1d0f7f7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:33,500 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:33,500 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=13, startTime=1732532133223; duration=0sec 2024-11-25T10:55:33,500 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:33,500 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:33,510 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:33,511 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/4e3bff3b9ff84a5eac64db74bd472fe6 is 50, key is test_row_0/C:col10/1732532132522/Put/seqid=0 2024-11-25T10:55:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:33,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741906_1082 (size=13051) 2024-11-25T10:55:33,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:33,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:33,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532193645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532193648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532193654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532193683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:33,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/6600c64731aa45ad943a566a05ed5405 2024-11-25T10:55:33,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/50aca871e8ee4eaea3bd0a1adcd19f9a is 50, key is test_row_0/B:col10/1732532133218/Put/seqid=0 2024-11-25T10:55:33,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741907_1083 (size=12301) 2024-11-25T10:55:33,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43616 deadline: 1732532193877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,888 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in 619a47d3e2a0b4c567d3fd0e3f1893d1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:55:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:33,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:33,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:33,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:33,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532193963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532193964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532193974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:33,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532193994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:34,009 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/4e3bff3b9ff84a5eac64db74bd472fe6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/4e3bff3b9ff84a5eac64db74bd472fe6 2024-11-25T10:55:34,037 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 4e3bff3b9ff84a5eac64db74bd472fe6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:34,037 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:34,037 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=13, startTime=1732532133224; duration=0sec 2024-11-25T10:55:34,037 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:34,037 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:34,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:34,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:34,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.77 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/50aca871e8ee4eaea3bd0a1adcd19f9a 2024-11-25T10:55:34,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/db7e14c37d1444a4a3c614238655a6eb is 50, key is test_row_0/C:col10/1732532133218/Put/seqid=0 2024-11-25T10:55:34,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741908_1084 (size=12301) 2024-11-25T10:55:34,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.77 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/db7e14c37d1444a4a3c614238655a6eb 2024-11-25T10:55:34,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/6600c64731aa45ad943a566a05ed5405 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6600c64731aa45ad943a566a05ed5405 2024-11-25T10:55:34,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:34,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:34,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6600c64731aa45ad943a566a05ed5405, entries=150, sequenceid=384, filesize=12.0 K 2024-11-25T10:55:34,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/50aca871e8ee4eaea3bd0a1adcd19f9a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/50aca871e8ee4eaea3bd0a1adcd19f9a 2024-11-25T10:55:34,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/50aca871e8ee4eaea3bd0a1adcd19f9a, entries=150, sequenceid=384, filesize=12.0 K 2024-11-25T10:55:34,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/db7e14c37d1444a4a3c614238655a6eb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/db7e14c37d1444a4a3c614238655a6eb 2024-11-25T10:55:34,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532194476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532194478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:34,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/db7e14c37d1444a4a3c614238655a6eb, entries=150, sequenceid=384, filesize=12.0 K 2024-11-25T10:55:34,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~105.56 KB/108090, heapSize ~297.81 KB/304960, currentSize=73.80 KB/75570 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1270ms, sequenceid=384, compaction requested=false 2024-11-25T10:55:34,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:34,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:34,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:55:34,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:34,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:34,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:34,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:34,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:34,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:34,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/0ae7385218884c929369b0d9165713cc is 50, key is test_row_0/A:col10/1732532133314/Put/seqid=0 2024-11-25T10:55:34,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741909_1085 (size=14741) 2024-11-25T10:55:34,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/0ae7385218884c929369b0d9165713cc 2024-11-25T10:55:34,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532194628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/60c4cda87c624ec6b815c9dff07ce485 is 50, key is test_row_0/B:col10/1732532133314/Put/seqid=0 2024-11-25T10:55:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:34,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532194643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741910_1086 (size=12301) 2024-11-25T10:55:34,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532194764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532194774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:34,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:34,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:34,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:34,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532194986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532194995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:35,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/60c4cda87c624ec6b815c9dff07ce485 2024-11-25T10:55:35,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/eba38750925647a6b980eea830c5423f is 50, key is test_row_0/C:col10/1732532133314/Put/seqid=0 2024-11-25T10:55:35,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741911_1087 (size=12301) 2024-11-25T10:55:35,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/eba38750925647a6b980eea830c5423f 2024-11-25T10:55:35,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/0ae7385218884c929369b0d9165713cc as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/0ae7385218884c929369b0d9165713cc 2024-11-25T10:55:35,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/0ae7385218884c929369b0d9165713cc, entries=200, sequenceid=402, filesize=14.4 K 2024-11-25T10:55:35,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/60c4cda87c624ec6b815c9dff07ce485 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/60c4cda87c624ec6b815c9dff07ce485 2024-11-25T10:55:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/60c4cda87c624ec6b815c9dff07ce485, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T10:55:35,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/eba38750925647a6b980eea830c5423f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/eba38750925647a6b980eea830c5423f 2024-11-25T10:55:35,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/eba38750925647a6b980eea830c5423f, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T10:55:35,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 765ms, sequenceid=402, compaction requested=true 2024-11-25T10:55:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:35,265 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 619a47d3e2a0b4c567d3fd0e3f1893d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:35,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:55:35,268 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:35,273 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:35,273 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/A is initiating minor compaction (all files) 2024-11-25T10:55:35,273 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/A in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,274 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9ac6547f03a467bac65e906b1d0f7f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6600c64731aa45ad943a566a05ed5405, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/0ae7385218884c929369b0d9165713cc] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=39.2 K 2024-11-25T10:55:35,277 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9ac6547f03a467bac65e906b1d0f7f7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732532131343 2024-11-25T10:55:35,278 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6600c64731aa45ad943a566a05ed5405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732532133196 2024-11-25T10:55:35,278 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae7385218884c929369b0d9165713cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732532133281 2024-11-25T10:55:35,284 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:35,285 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/B is initiating minor compaction (all files) 2024-11-25T10:55:35,285 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/B in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,285 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7bf99ea59c334404b635b9bb2651e027, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/50aca871e8ee4eaea3bd0a1adcd19f9a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/60c4cda87c624ec6b815c9dff07ce485] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.8 K 2024-11-25T10:55:35,292 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf99ea59c334404b635b9bb2651e027, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732532131343 2024-11-25T10:55:35,296 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 50aca871e8ee4eaea3bd0a1adcd19f9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732532133196 2024-11-25T10:55:35,304 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 60c4cda87c624ec6b815c9dff07ce485, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732532133306 2024-11-25T10:55:35,312 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:35,317 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a7c38d24054f48b3b39a219a4c566172 is 50, key is test_row_0/A:col10/1732532133314/Put/seqid=0 2024-11-25T10:55:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:35,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:55:35,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:35,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:35,341 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:35,342 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/f30de27fdbad40748d32e1ae3d0745ec is 50, key is test_row_0/B:col10/1732532133314/Put/seqid=0 2024-11-25T10:55:35,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741912_1088 (size=13153) 2024-11-25T10:55:35,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/3fc810b14c744a7b939e62fe57f6e5a0 is 50, key is test_row_0/A:col10/1732532135325/Put/seqid=0 2024-11-25T10:55:35,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:35,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:35,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741913_1089 (size=13153) 2024-11-25T10:55:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532195402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741914_1090 (size=14741) 2024-11-25T10:55:35,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532195411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,430 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/f30de27fdbad40748d32e1ae3d0745ec as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/f30de27fdbad40748d32e1ae3d0745ec 2024-11-25T10:55:35,451 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/B of 619a47d3e2a0b4c567d3fd0e3f1893d1 into f30de27fdbad40748d32e1ae3d0745ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:35,451 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:35,452 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/B, priority=13, startTime=1732532135265; duration=0sec 2024-11-25T10:55:35,452 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:35,452 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:B 2024-11-25T10:55:35,452 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:35,454 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:35,455 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 619a47d3e2a0b4c567d3fd0e3f1893d1/C is initiating minor compaction (all files) 2024-11-25T10:55:35,455 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 619a47d3e2a0b4c567d3fd0e3f1893d1/C in TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,455 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/4e3bff3b9ff84a5eac64db74bd472fe6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/db7e14c37d1444a4a3c614238655a6eb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/eba38750925647a6b980eea830c5423f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp, totalSize=36.8 K 2024-11-25T10:55:35,456 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e3bff3b9ff84a5eac64db74bd472fe6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732532131343 2024-11-25T10:55:35,460 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting db7e14c37d1444a4a3c614238655a6eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732532133196 2024-11-25T10:55:35,465 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting eba38750925647a6b980eea830c5423f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732532133306 2024-11-25T10:55:35,511 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 619a47d3e2a0b4c567d3fd0e3f1893d1#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:35,512 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/2e3a4aa1ede4441fa5ce7c572ce11138 is 50, key is test_row_0/C:col10/1732532133314/Put/seqid=0 2024-11-25T10:55:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43630 deadline: 1732532195506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,516 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in 619a47d3e2a0b4c567d3fd0e3f1893d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:55:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43684 deadline: 1732532195506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532195513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,517 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in 619a47d3e2a0b4c567d3fd0e3f1893d1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:55:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532195534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741915_1091 (size=13153) 2024-11-25T10:55:35,583 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/2e3a4aa1ede4441fa5ce7c572ce11138 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/2e3a4aa1ede4441fa5ce7c572ce11138 2024-11-25T10:55:35,593 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/C of 619a47d3e2a0b4c567d3fd0e3f1893d1 into 2e3a4aa1ede4441fa5ce7c572ce11138(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:35,593 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:35,593 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/C, priority=13, startTime=1732532135265; duration=0sec 2024-11-25T10:55:35,593 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:35,593 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:C 2024-11-25T10:55:35,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532195734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532195744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:35,779 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/a7c38d24054f48b3b39a219a4c566172 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7c38d24054f48b3b39a219a4c566172 2024-11-25T10:55:35,798 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 619a47d3e2a0b4c567d3fd0e3f1893d1/A of 619a47d3e2a0b4c567d3fd0e3f1893d1 into a7c38d24054f48b3b39a219a4c566172(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:35,798 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:35,798 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., storeName=619a47d3e2a0b4c567d3fd0e3f1893d1/A, priority=13, startTime=1732532135265; duration=0sec 2024-11-25T10:55:35,799 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:35,799 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 619a47d3e2a0b4c567d3fd0e3f1893d1:A 2024-11-25T10:55:35,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/3fc810b14c744a7b939e62fe57f6e5a0 2024-11-25T10:55:35,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/6675085a1d864d35a927b06c1efa8620 is 50, key is test_row_0/B:col10/1732532135325/Put/seqid=0 2024-11-25T10:55:35,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:35,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:35,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:35,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:35,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741916_1092 (size=12301) 2024-11-25T10:55:36,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:36,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:36,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532196055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:36,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:36,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532196067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:36,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:36,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/6675085a1d864d35a927b06c1efa8620 2024-11-25T10:55:36,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/b06b7ca44e1043559854144a646d3f6b is 50, key is test_row_0/C:col10/1732532135325/Put/seqid=0 2024-11-25T10:55:36,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741917_1093 (size=12301) 2024-11-25T10:55:36,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:36,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:36,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43658 deadline: 1732532196573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:36,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43644 deadline: 1732532196584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:55:36,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. as already flushing 2024-11-25T10:55:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:36,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:36,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/b06b7ca44e1043559854144a646d3f6b 2024-11-25T10:55:36,865 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-25T10:55:36,865 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:36,867 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-25T10:55:36,867 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:36,869 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-25T10:55:36,869 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:36,872 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,873 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:36,876 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,877 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:36,877 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,877 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:36,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/3fc810b14c744a7b939e62fe57f6e5a0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3fc810b14c744a7b939e62fe57f6e5a0 2024-11-25T10:55:36,891 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-25T10:55:36,891 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:36,891 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,891 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:36,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3fc810b14c744a7b939e62fe57f6e5a0, entries=200, sequenceid=421, filesize=14.4 K 2024-11-25T10:55:36,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:36,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/6675085a1d864d35a927b06c1efa8620 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/6675085a1d864d35a927b06c1efa8620 2024-11-25T10:55:36,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/6675085a1d864d35a927b06c1efa8620, entries=150, sequenceid=421, filesize=12.0 K 2024-11-25T10:55:36,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/b06b7ca44e1043559854144a646d3f6b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b06b7ca44e1043559854144a646d3f6b 2024-11-25T10:55:36,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b06b7ca44e1043559854144a646d3f6b, entries=150, sequenceid=421, filesize=12.0 K 2024-11-25T10:55:36,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1586ms, sequenceid=421, compaction requested=false 2024-11-25T10:55:36,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:37,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T10:55:37,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:37,003 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:55:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:37,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/237e779424004424b7587f3c99627d24 is 50, key is test_row_0/A:col10/1732532135398/Put/seqid=0 2024-11-25T10:55:37,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741918_1094 (size=12301) 2024-11-25T10:55:37,070 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/237e779424004424b7587f3c99627d24 2024-11-25T10:55:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/3c010ed1be084f12942bf563cd3b53cb is 50, key is test_row_0/B:col10/1732532135398/Put/seqid=0 2024-11-25T10:55:37,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741919_1095 (size=12301) 2024-11-25T10:55:37,233 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/3c010ed1be084f12942bf563cd3b53cb 2024-11-25T10:55:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6fbdaf68332a4112bd4ed22630a0ef1a is 50, key is test_row_0/C:col10/1732532135398/Put/seqid=0 2024-11-25T10:55:37,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741920_1096 (size=12301) 2024-11-25T10:55:37,302 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6fbdaf68332a4112bd4ed22630a0ef1a 2024-11-25T10:55:37,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/237e779424004424b7587f3c99627d24 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/237e779424004424b7587f3c99627d24 2024-11-25T10:55:37,341 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/237e779424004424b7587f3c99627d24, entries=150, sequenceid=437, filesize=12.0 K 2024-11-25T10:55:37,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/3c010ed1be084f12942bf563cd3b53cb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/3c010ed1be084f12942bf563cd3b53cb 2024-11-25T10:55:37,350 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/3c010ed1be084f12942bf563cd3b53cb, entries=150, sequenceid=437, filesize=12.0 K 2024-11-25T10:55:37,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/6fbdaf68332a4112bd4ed22630a0ef1a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6fbdaf68332a4112bd4ed22630a0ef1a 2024-11-25T10:55:37,362 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6fbdaf68332a4112bd4ed22630a0ef1a, entries=150, sequenceid=437, filesize=12.0 K 2024-11-25T10:55:37,364 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 360ms, sequenceid=437, compaction requested=true 2024-11-25T10:55:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: 2024-11-25T10:55:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:37,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-25T10:55:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-25T10:55:37,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T10:55:37,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4260 sec 2024-11-25T10:55:37,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 4.4610 sec 2024-11-25T10:55:37,465 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T10:55:37,546 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-25T10:55:37,546 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:37,546 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,546 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,547 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-25T10:55:37,547 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:37,547 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,547 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:37,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:37,597 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-25T10:55:37,597 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:37,597 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,597 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:37,621 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-25T10:55:37,621 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:37,621 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,621 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:37,928 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-25T10:55:37,928 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:55:37,928 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,928 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:37,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T10:55:41,114 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:41,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T10:55:41,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-25T10:55:41,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-25T10:55:41,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-25T10:55:41,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1237 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1247 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 635 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1904 rows 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 631 2024-11-25T10:55:41,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1892 rows 2024-11-25T10:55:41,115 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:55:41,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:55:41,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:41,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:41,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:55:41,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T10:55:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:41,137 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532141137"}]},"ts":"1732532141137"} 2024-11-25T10:55:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T10:55:41,149 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T10:55:41,149 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T10:55:41,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:55:41,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, UNASSIGN}] 2024-11-25T10:55:41,181 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, UNASSIGN 2024-11-25T10:55:41,194 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=619a47d3e2a0b4c567d3fd0e3f1893d1, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:41,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, UNASSIGN because future has completed 2024-11-25T10:55:41,200 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:55:41,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:55:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T10:55:41,365 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:41,365 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:55:41,366 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 619a47d3e2a0b4c567d3fd0e3f1893d1, disabling compactions & flushes 2024-11-25T10:55:41,366 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. after waiting 0 ms 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:41,367 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=A 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=B 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:41,367 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 619a47d3e2a0b4c567d3fd0e3f1893d1, store=C 2024-11-25T10:55:41,368 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:41,381 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/1afe90ac165f4ea7a75250d3fa119032 is 50, key is test_row_0/A:col10/1732532137537/Put/seqid=0 2024-11-25T10:55:41,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741921_1097 (size=12301) 2024-11-25T10:55:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T10:55:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T10:55:41,824 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/1afe90ac165f4ea7a75250d3fa119032 2024-11-25T10:55:41,867 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/712b87c3392d472f9dbea6e05380e38d is 50, key is test_row_0/B:col10/1732532137537/Put/seqid=0 2024-11-25T10:55:41,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741922_1098 (size=12301) 2024-11-25T10:55:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T10:55:42,312 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/712b87c3392d472f9dbea6e05380e38d 2024-11-25T10:55:42,347 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3768a52f675946d993dcae52184946c1 is 50, key is test_row_0/C:col10/1732532137537/Put/seqid=0 2024-11-25T10:55:42,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741923_1099 (size=12301) 2024-11-25T10:55:42,393 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3768a52f675946d993dcae52184946c1 2024-11-25T10:55:42,403 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/A/1afe90ac165f4ea7a75250d3fa119032 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/1afe90ac165f4ea7a75250d3fa119032 2024-11-25T10:55:42,418 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/1afe90ac165f4ea7a75250d3fa119032, entries=150, sequenceid=445, filesize=12.0 K 2024-11-25T10:55:42,420 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/B/712b87c3392d472f9dbea6e05380e38d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/712b87c3392d472f9dbea6e05380e38d 2024-11-25T10:55:42,437 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/712b87c3392d472f9dbea6e05380e38d, entries=150, sequenceid=445, filesize=12.0 K 2024-11-25T10:55:42,440 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/.tmp/C/3768a52f675946d993dcae52184946c1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3768a52f675946d993dcae52184946c1 2024-11-25T10:55:42,474 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3768a52f675946d993dcae52184946c1, entries=150, sequenceid=445, filesize=12.0 K 2024-11-25T10:55:42,478 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1111ms, sequenceid=445, compaction requested=true 2024-11-25T10:55:42,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9f54d01422649ee973ad4f4825f9b2e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/f87ff97a1c294729b02f05b23f2586a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/492d8089c03e43ffa120de2d3f4a633d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/8816887003784638a5c2502333197019, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/986e4edbfb874e2ab467b243593051c9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/c920a46673ad4c378de8891c505907e8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/7fcdfb01e7fc4ac3b948469116cab7f9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3bc57fac3a224696bac99ed5fc14563c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/de7991385e8f4f75a0aef956d8fa6b76, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5f6c8749163d4f95b6e7e4f6a467d9dc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/9b89b874be3b4684888fa63d4a8780ae, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/be3b9e43ea7944dd98e0907c64ea3b7b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/46d98ce98dd04f82a85dc339c4dac9d1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d3676e01b11e479ca093e94beb5b4c59, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/e40700c489974f36b051768e86cc4517, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6c9bbe677e6443468556a6c687851e41, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d4dd8df0936746d5a3549fec8ccb7524, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5c48aa020ea941e4ac8dfe4d57c68fd9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/444d8b898a6d4cdc9b003990480bfc68, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/553e054a6e3a449b8530cbcc721bdf6e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9ac6547f03a467bac65e906b1d0f7f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/cebfe3e5878a447fa5ae556ab4b82bf8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6600c64731aa45ad943a566a05ed5405, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/0ae7385218884c929369b0d9165713cc] to archive 2024-11-25T10:55:42,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:55:42,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9f54d01422649ee973ad4f4825f9b2e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9f54d01422649ee973ad4f4825f9b2e 2024-11-25T10:55:42,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/f87ff97a1c294729b02f05b23f2586a3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/f87ff97a1c294729b02f05b23f2586a3 2024-11-25T10:55:42,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/492d8089c03e43ffa120de2d3f4a633d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/492d8089c03e43ffa120de2d3f4a633d 2024-11-25T10:55:42,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/8816887003784638a5c2502333197019 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/8816887003784638a5c2502333197019 2024-11-25T10:55:42,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/986e4edbfb874e2ab467b243593051c9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/986e4edbfb874e2ab467b243593051c9 2024-11-25T10:55:42,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/c920a46673ad4c378de8891c505907e8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/c920a46673ad4c378de8891c505907e8 2024-11-25T10:55:42,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/7fcdfb01e7fc4ac3b948469116cab7f9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/7fcdfb01e7fc4ac3b948469116cab7f9 2024-11-25T10:55:42,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3bc57fac3a224696bac99ed5fc14563c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3bc57fac3a224696bac99ed5fc14563c 2024-11-25T10:55:42,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/de7991385e8f4f75a0aef956d8fa6b76 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/de7991385e8f4f75a0aef956d8fa6b76 2024-11-25T10:55:42,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5f6c8749163d4f95b6e7e4f6a467d9dc to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5f6c8749163d4f95b6e7e4f6a467d9dc 2024-11-25T10:55:42,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/9b89b874be3b4684888fa63d4a8780ae to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/9b89b874be3b4684888fa63d4a8780ae 2024-11-25T10:55:42,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/be3b9e43ea7944dd98e0907c64ea3b7b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/be3b9e43ea7944dd98e0907c64ea3b7b 2024-11-25T10:55:42,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/46d98ce98dd04f82a85dc339c4dac9d1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/46d98ce98dd04f82a85dc339c4dac9d1 2024-11-25T10:55:42,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d3676e01b11e479ca093e94beb5b4c59 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d3676e01b11e479ca093e94beb5b4c59 2024-11-25T10:55:42,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7d3a1ee94b54cf6ad6dc8006d63fb2d 2024-11-25T10:55:42,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/e40700c489974f36b051768e86cc4517 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/e40700c489974f36b051768e86cc4517 2024-11-25T10:55:42,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6c9bbe677e6443468556a6c687851e41 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6c9bbe677e6443468556a6c687851e41 2024-11-25T10:55:42,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d4dd8df0936746d5a3549fec8ccb7524 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/d4dd8df0936746d5a3549fec8ccb7524 2024-11-25T10:55:42,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5c48aa020ea941e4ac8dfe4d57c68fd9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/5c48aa020ea941e4ac8dfe4d57c68fd9 2024-11-25T10:55:42,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/444d8b898a6d4cdc9b003990480bfc68 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/444d8b898a6d4cdc9b003990480bfc68 2024-11-25T10:55:42,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/553e054a6e3a449b8530cbcc721bdf6e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/553e054a6e3a449b8530cbcc721bdf6e 2024-11-25T10:55:42,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9ac6547f03a467bac65e906b1d0f7f7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a9ac6547f03a467bac65e906b1d0f7f7 2024-11-25T10:55:42,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/cebfe3e5878a447fa5ae556ab4b82bf8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/cebfe3e5878a447fa5ae556ab4b82bf8 2024-11-25T10:55:42,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6600c64731aa45ad943a566a05ed5405 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/6600c64731aa45ad943a566a05ed5405 2024-11-25T10:55:42,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/0ae7385218884c929369b0d9165713cc to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/0ae7385218884c929369b0d9165713cc 2024-11-25T10:55:42,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/5d135bba5cc0447895a99682bd13a6f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/dafe5af42b3b4ead9bdc82648527f0b2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ca3016a8e7643278c3bb1c82c05f318, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/538cfbea6c8749a683a9711ec1bf3499, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/29e8ba68d2344330a1aab25f8a332bdf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0ee8c33948c74eaabb312892eb04f63e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/65b0b0f6a44347ac9240182da8c413ee, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1ff6629fe13c41bf9cb6eac035261b20, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/52feeba6d70c47f3aa13362c686fa7a6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0cae9044b2f1496f8eafd1a93059bf83, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc5f0cce115140f1b9869dc23ea7bc33, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/ba5ce059ee594efea2f6c3e6094e395b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/38a5fb13aad34c5e9944b46155882b4f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/805aab0cd6cb40199291666511abb4f9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/814bd5a0db37438bb3aa1ee073228587, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/05ed2d7cb17d40c481169583538dabd9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/66815f9fd2ae4b08a2df30becbefb86b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ea063909ae246da961f2b654805f986, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc1f426644314e0381c41ae0cb77cc38, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7e8764d217294ec98f4bc88c68e1fff2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7fee6b2ab12d45339319c5522b26e5c6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7bf99ea59c334404b635b9bb2651e027, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1d797c3a9e7740729812c16943e96c7f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/50aca871e8ee4eaea3bd0a1adcd19f9a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/60c4cda87c624ec6b815c9dff07ce485] to archive 2024-11-25T10:55:42,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:55:42,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/5d135bba5cc0447895a99682bd13a6f7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/5d135bba5cc0447895a99682bd13a6f7 2024-11-25T10:55:42,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/dafe5af42b3b4ead9bdc82648527f0b2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/dafe5af42b3b4ead9bdc82648527f0b2 2024-11-25T10:55:42,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ca3016a8e7643278c3bb1c82c05f318 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ca3016a8e7643278c3bb1c82c05f318 2024-11-25T10:55:42,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/538cfbea6c8749a683a9711ec1bf3499 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/538cfbea6c8749a683a9711ec1bf3499 2024-11-25T10:55:42,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/29e8ba68d2344330a1aab25f8a332bdf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/29e8ba68d2344330a1aab25f8a332bdf 2024-11-25T10:55:42,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0ee8c33948c74eaabb312892eb04f63e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0ee8c33948c74eaabb312892eb04f63e 2024-11-25T10:55:42,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/65b0b0f6a44347ac9240182da8c413ee to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/65b0b0f6a44347ac9240182da8c413ee 2024-11-25T10:55:42,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1ff6629fe13c41bf9cb6eac035261b20 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1ff6629fe13c41bf9cb6eac035261b20 2024-11-25T10:55:42,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/52feeba6d70c47f3aa13362c686fa7a6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/52feeba6d70c47f3aa13362c686fa7a6 2024-11-25T10:55:42,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0cae9044b2f1496f8eafd1a93059bf83 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/0cae9044b2f1496f8eafd1a93059bf83 2024-11-25T10:55:42,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc5f0cce115140f1b9869dc23ea7bc33 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc5f0cce115140f1b9869dc23ea7bc33 2024-11-25T10:55:42,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/ba5ce059ee594efea2f6c3e6094e395b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/ba5ce059ee594efea2f6c3e6094e395b 2024-11-25T10:55:42,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/38a5fb13aad34c5e9944b46155882b4f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/38a5fb13aad34c5e9944b46155882b4f 2024-11-25T10:55:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/805aab0cd6cb40199291666511abb4f9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/805aab0cd6cb40199291666511abb4f9 2024-11-25T10:55:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/814bd5a0db37438bb3aa1ee073228587 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/814bd5a0db37438bb3aa1ee073228587 2024-11-25T10:55:42,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/05ed2d7cb17d40c481169583538dabd9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/05ed2d7cb17d40c481169583538dabd9 2024-11-25T10:55:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/66815f9fd2ae4b08a2df30becbefb86b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/66815f9fd2ae4b08a2df30becbefb86b 2024-11-25T10:55:42,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ea063909ae246da961f2b654805f986 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7ea063909ae246da961f2b654805f986 2024-11-25T10:55:42,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc1f426644314e0381c41ae0cb77cc38 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/bc1f426644314e0381c41ae0cb77cc38 2024-11-25T10:55:42,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7e8764d217294ec98f4bc88c68e1fff2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7e8764d217294ec98f4bc88c68e1fff2 2024-11-25T10:55:42,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7fee6b2ab12d45339319c5522b26e5c6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7fee6b2ab12d45339319c5522b26e5c6 2024-11-25T10:55:42,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7bf99ea59c334404b635b9bb2651e027 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/7bf99ea59c334404b635b9bb2651e027 2024-11-25T10:55:42,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1d797c3a9e7740729812c16943e96c7f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/1d797c3a9e7740729812c16943e96c7f 2024-11-25T10:55:42,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/50aca871e8ee4eaea3bd0a1adcd19f9a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/50aca871e8ee4eaea3bd0a1adcd19f9a 2024-11-25T10:55:42,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/60c4cda87c624ec6b815c9dff07ce485 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/60c4cda87c624ec6b815c9dff07ce485 2024-11-25T10:55:42,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3a5bb642c84a44368ad414e8b177bf66, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6525a836ce3c462fb766406a53a6f927, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/f4be4bc7ce584db695473f6830b118ee, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/ebb41d13ec6c46f58e52d90abe5716ad, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/be4ebe0e976145c2a31ec7af9e7b0d70, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/df64a57688e8492bba9d7e5d379eba5c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/0db9118d0080454d932a995c9d8e18f4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/fa1b9b5f0a3b42d8a0005e6f051bd004, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/8ae327c189c446658618d978a49af519, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d23f82ab2cae46ae8c3751ad2efd0294, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a3b1e0dd54014a7d94a53e052e1ea9f0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/9526ed94c77349208c527837805b4dde, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/7740132fc6e849a99661876fdbf83db9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b5d4c2c54b5f44e88e3909a2d2a2d482, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd5d5aedd6a64cff823569a5accf4ef6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/37e8e4af8e1e4a7384acc15735a2db5f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/18f10eb0c1694138af8e96d451c41a55, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3ca9c3c15893489d9b14e847a902c82c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6f26bfa60706407db833d276586b985e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d72aa8f7132941bb92a6f66ada6fa41e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a1c9071575d14af8bf915287833809bc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/4e3bff3b9ff84a5eac64db74bd472fe6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd4ea06198b4456892165ca83a51eb0d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/db7e14c37d1444a4a3c614238655a6eb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/eba38750925647a6b980eea830c5423f] to archive 2024-11-25T10:55:42,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:55:42,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3a5bb642c84a44368ad414e8b177bf66 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3a5bb642c84a44368ad414e8b177bf66 2024-11-25T10:55:42,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6525a836ce3c462fb766406a53a6f927 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6525a836ce3c462fb766406a53a6f927 2024-11-25T10:55:42,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/f4be4bc7ce584db695473f6830b118ee to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/f4be4bc7ce584db695473f6830b118ee 2024-11-25T10:55:42,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/ebb41d13ec6c46f58e52d90abe5716ad to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/ebb41d13ec6c46f58e52d90abe5716ad 2024-11-25T10:55:42,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/be4ebe0e976145c2a31ec7af9e7b0d70 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/be4ebe0e976145c2a31ec7af9e7b0d70 2024-11-25T10:55:42,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/df64a57688e8492bba9d7e5d379eba5c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/df64a57688e8492bba9d7e5d379eba5c 2024-11-25T10:55:42,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/0db9118d0080454d932a995c9d8e18f4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/0db9118d0080454d932a995c9d8e18f4 2024-11-25T10:55:42,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/fa1b9b5f0a3b42d8a0005e6f051bd004 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/fa1b9b5f0a3b42d8a0005e6f051bd004 2024-11-25T10:55:42,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/8ae327c189c446658618d978a49af519 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/8ae327c189c446658618d978a49af519 2024-11-25T10:55:42,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d23f82ab2cae46ae8c3751ad2efd0294 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d23f82ab2cae46ae8c3751ad2efd0294 2024-11-25T10:55:42,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a3b1e0dd54014a7d94a53e052e1ea9f0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a3b1e0dd54014a7d94a53e052e1ea9f0 2024-11-25T10:55:42,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/9526ed94c77349208c527837805b4dde to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/9526ed94c77349208c527837805b4dde 2024-11-25T10:55:42,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/7740132fc6e849a99661876fdbf83db9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/7740132fc6e849a99661876fdbf83db9 2024-11-25T10:55:42,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b5d4c2c54b5f44e88e3909a2d2a2d482 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b5d4c2c54b5f44e88e3909a2d2a2d482 2024-11-25T10:55:42,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd5d5aedd6a64cff823569a5accf4ef6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd5d5aedd6a64cff823569a5accf4ef6 2024-11-25T10:55:42,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/37e8e4af8e1e4a7384acc15735a2db5f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/37e8e4af8e1e4a7384acc15735a2db5f 2024-11-25T10:55:42,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/18f10eb0c1694138af8e96d451c41a55 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/18f10eb0c1694138af8e96d451c41a55 2024-11-25T10:55:42,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3ca9c3c15893489d9b14e847a902c82c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3ca9c3c15893489d9b14e847a902c82c 2024-11-25T10:55:42,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6f26bfa60706407db833d276586b985e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6f26bfa60706407db833d276586b985e 2024-11-25T10:55:43,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d72aa8f7132941bb92a6f66ada6fa41e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/d72aa8f7132941bb92a6f66ada6fa41e 2024-11-25T10:55:43,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a1c9071575d14af8bf915287833809bc to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/a1c9071575d14af8bf915287833809bc 2024-11-25T10:55:43,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/4e3bff3b9ff84a5eac64db74bd472fe6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/4e3bff3b9ff84a5eac64db74bd472fe6 2024-11-25T10:55:43,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd4ea06198b4456892165ca83a51eb0d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/cd4ea06198b4456892165ca83a51eb0d 2024-11-25T10:55:43,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/db7e14c37d1444a4a3c614238655a6eb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/db7e14c37d1444a4a3c614238655a6eb 2024-11-25T10:55:43,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/eba38750925647a6b980eea830c5423f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/eba38750925647a6b980eea830c5423f 2024-11-25T10:55:43,038 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/recovered.edits/448.seqid, newMaxSeqId=448, maxSeqId=1 2024-11-25T10:55:43,040 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. 2024-11-25T10:55:43,040 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 619a47d3e2a0b4c567d3fd0e3f1893d1: Waiting for close lock at 1732532141366Running coprocessor pre-close hooks at 1732532141366Disabling compacts and flushes for region at 1732532141366Disabling writes for close at 1732532141367 (+1 ms)Obtaining lock to block concurrent updates at 1732532141367Preparing flush snapshotting stores in 619a47d3e2a0b4c567d3fd0e3f1893d1 at 1732532141367Finished memstore snapshotting TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732532141368 (+1 ms)Flushing stores of TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1. at 1732532141369 (+1 ms)Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/A: creating writer at 1732532141369Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/A: appending metadata at 1732532141381 (+12 ms)Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/A: closing flushed file at 1732532141381Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/B: creating writer at 1732532141842 (+461 ms)Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/B: appending metadata at 1732532141866 (+24 ms)Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/B: closing flushed file at 1732532141866Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/C: creating writer at 1732532142332 (+466 ms)Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/C: appending metadata at 1732532142346 (+14 ms)Flushing 619a47d3e2a0b4c567d3fd0e3f1893d1/C: closing flushed file at 1732532142346Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62b60145: reopening flushed file at 1732532142400 (+54 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77e5e0f8: reopening flushed file at 1732532142418 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1646eb6: reopening flushed file at 1732532142438 (+20 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 619a47d3e2a0b4c567d3fd0e3f1893d1 in 1111ms, sequenceid=445, compaction requested=true at 1732532142478 (+40 ms)Writing region close event to WAL at 1732532143032 (+554 ms)Running coprocessor post-close hooks at 1732532143039 (+7 ms)Closed at 1732532143039 2024-11-25T10:55:43,047 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:43,047 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=619a47d3e2a0b4c567d3fd0e3f1893d1, regionState=CLOSED 2024-11-25T10:55:43,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:55:43,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T10:55:43,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 619a47d3e2a0b4c567d3fd0e3f1893d1, server=6bfc37bad115,44671,1732532110409 in 1.8500 sec 2024-11-25T10:55:43,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-25T10:55:43,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=619a47d3e2a0b4c567d3fd0e3f1893d1, UNASSIGN in 1.8790 sec 2024-11-25T10:55:43,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-25T10:55:43,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9050 sec 2024-11-25T10:55:43,060 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532143060"}]},"ts":"1732532143060"} 2024-11-25T10:55:43,064 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T10:55:43,064 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T10:55:43,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9360 sec 2024-11-25T10:55:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T10:55:43,314 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:43,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T10:55:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:43,330 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:43,333 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T10:55:43,360 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:43,387 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/recovered.edits] 2024-11-25T10:55:43,408 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/1afe90ac165f4ea7a75250d3fa119032 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/1afe90ac165f4ea7a75250d3fa119032 2024-11-25T10:55:43,420 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/237e779424004424b7587f3c99627d24 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/237e779424004424b7587f3c99627d24 2024-11-25T10:55:43,437 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3fc810b14c744a7b939e62fe57f6e5a0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/3fc810b14c744a7b939e62fe57f6e5a0 2024-11-25T10:55:43,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T10:55:43,446 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7c38d24054f48b3b39a219a4c566172 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/A/a7c38d24054f48b3b39a219a4c566172 2024-11-25T10:55:43,462 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/3c010ed1be084f12942bf563cd3b53cb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/3c010ed1be084f12942bf563cd3b53cb 2024-11-25T10:55:43,470 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/6675085a1d864d35a927b06c1efa8620 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/6675085a1d864d35a927b06c1efa8620 2024-11-25T10:55:43,473 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/712b87c3392d472f9dbea6e05380e38d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/712b87c3392d472f9dbea6e05380e38d 2024-11-25T10:55:43,476 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/f30de27fdbad40748d32e1ae3d0745ec to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/B/f30de27fdbad40748d32e1ae3d0745ec 2024-11-25T10:55:43,480 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/2e3a4aa1ede4441fa5ce7c572ce11138 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/2e3a4aa1ede4441fa5ce7c572ce11138 2024-11-25T10:55:43,482 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3768a52f675946d993dcae52184946c1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/3768a52f675946d993dcae52184946c1 2024-11-25T10:55:43,486 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6fbdaf68332a4112bd4ed22630a0ef1a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/6fbdaf68332a4112bd4ed22630a0ef1a 2024-11-25T10:55:43,488 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b06b7ca44e1043559854144a646d3f6b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/C/b06b7ca44e1043559854144a646d3f6b 2024-11-25T10:55:43,498 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/recovered.edits/448.seqid to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1/recovered.edits/448.seqid 2024-11-25T10:55:43,499 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/619a47d3e2a0b4c567d3fd0e3f1893d1 2024-11-25T10:55:43,500 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T10:55:43,520 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:43,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-25T10:55:43,543 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T10:55:43,622 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T10:55:43,626 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:43,627 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T10:55:43,627 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732532143627"}]},"ts":"9223372036854775807"} 2024-11-25T10:55:43,638 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T10:55:43,638 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 619a47d3e2a0b4c567d3fd0e3f1893d1, NAME => 'TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T10:55:43,638 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T10:55:43,639 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732532143638"}]},"ts":"9223372036854775807"} 2024-11-25T10:55:43,645 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T10:55:43,646 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:43,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 325 msec 2024-11-25T10:55:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T10:55:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T10:55:43,665 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:43,726 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=246 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;6bfc37bad115:44671-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1892856258_22 at /127.0.0.1:45268 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1231 (was 1017) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 12), AvailableMemoryMB=1418 (was 731) - AvailableMemoryMB LEAK? - 2024-11-25T10:55:43,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1231, ProcessCount=11, AvailableMemoryMB=1403 2024-11-25T10:55:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:55:43,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:55:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:43,769 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T10:55:43,769 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:43,770 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T10:55:43,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-25T10:55:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T10:55:43,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741924_1100 (size=1061) 2024-11-25T10:55:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T10:55:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T10:55:44,253 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:55:44,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741925_1101 (size=53) 2024-11-25T10:55:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T10:55:44,710 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:44,710 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 78f10d0299f2958c87d34c58cfcb3253, disabling compactions & flushes 2024-11-25T10:55:44,710 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:44,710 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:44,710 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. after waiting 0 ms 2024-11-25T10:55:44,710 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:44,710 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:44,710 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 78f10d0299f2958c87d34c58cfcb3253: Waiting for close lock at 1732532144710Disabling compacts and flushes for region at 1732532144710Disabling writes for close at 1732532144710Writing region close event to WAL at 1732532144710Closed at 1732532144710 2024-11-25T10:55:44,721 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T10:55:44,721 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732532144721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532144721"}]},"ts":"1732532144721"} 2024-11-25T10:55:44,736 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T10:55:44,749 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T10:55:44,749 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532144749"}]},"ts":"1732532144749"} 2024-11-25T10:55:44,759 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T10:55:44,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, ASSIGN}] 2024-11-25T10:55:44,767 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, ASSIGN 2024-11-25T10:55:44,769 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, ASSIGN; state=OFFLINE, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=false 2024-11-25T10:55:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T10:55:44,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:44,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, ASSIGN because future has completed 2024-11-25T10:55:44,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:55:45,105 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:45,106 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:55:45,106 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,106 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:45,106 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,107 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,121 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,132 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:45,133 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78f10d0299f2958c87d34c58cfcb3253 columnFamilyName A 2024-11-25T10:55:45,133 DEBUG [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:45,140 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(327): Store=78f10d0299f2958c87d34c58cfcb3253/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:45,141 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,160 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:45,161 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78f10d0299f2958c87d34c58cfcb3253 columnFamilyName B 2024-11-25T10:55:45,161 DEBUG [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:45,165 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(327): Store=78f10d0299f2958c87d34c58cfcb3253/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:45,165 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,173 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:45,173 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78f10d0299f2958c87d34c58cfcb3253 columnFamilyName C 2024-11-25T10:55:45,173 DEBUG [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:45,179 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(327): Store=78f10d0299f2958c87d34c58cfcb3253/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:45,179 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:45,179 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,180 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,181 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,182 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,182 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,183 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:55:45,189 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,199 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:55:45,200 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 78f10d0299f2958c87d34c58cfcb3253; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67581747, jitterRate=0.007046505808830261}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:55:45,200 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:45,201 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 78f10d0299f2958c87d34c58cfcb3253: Running coprocessor pre-open hook at 1732532145107Writing region info on filesystem at 1732532145107Initializing all the Stores at 1732532145112 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532145112Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532145113 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532145113Cleaning up temporary data from old regions at 1732532145182 (+69 ms)Running coprocessor post-open hooks at 1732532145200 (+18 ms)Region opened successfully at 1732532145201 (+1 ms) 2024-11-25T10:55:45,205 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., pid=28, masterSystemTime=1732532145091 2024-11-25T10:55:45,221 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:45,225 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:45,226 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:45,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:55:45,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-25T10:55:45,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 in 294 msec 2024-11-25T10:55:45,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-25T10:55:45,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, ASSIGN in 475 msec 2024-11-25T10:55:45,240 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T10:55:45,240 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532145240"}]},"ts":"1732532145240"} 2024-11-25T10:55:45,245 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T10:55:45,246 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T10:55:45,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.4820 sec 2024-11-25T10:55:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T10:55:45,934 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:45,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff3539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:45,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:45,935 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:45,938 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c28280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:45,940 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a98538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:45,945 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:45,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:45,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T10:55:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39013,1732532109078 2024-11-25T10:55:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f794af 2024-11-25T10:55:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T10:55:45,955 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T10:55:45,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:55:45,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:55:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T10:55:45,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741926_1102 (size=1097) 2024-11-25T10:55:46,399 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T10:55:46,399 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T10:55:46,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:55:46,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, REOPEN/MOVE}] 2024-11-25T10:55:46,417 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, REOPEN/MOVE 2024-11-25T10:55:46,418 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:46,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, REOPEN/MOVE because future has completed 2024-11-25T10:55:46,422 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:55:46,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:55:46,577 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:46,577 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:55:46,578 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 78f10d0299f2958c87d34c58cfcb3253, disabling compactions & flushes 2024-11-25T10:55:46,578 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:46,578 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:46,578 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. after waiting 0 ms 2024-11-25T10:55:46,578 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:46,607 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T10:55:46,613 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:46,613 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 78f10d0299f2958c87d34c58cfcb3253: Waiting for close lock at 1732532146578Running coprocessor pre-close hooks at 1732532146578Disabling compacts and flushes for region at 1732532146578Disabling writes for close at 1732532146578Writing region close event to WAL at 1732532146581 (+3 ms)Running coprocessor post-close hooks at 1732532146613 (+32 ms)Closed at 1732532146613 2024-11-25T10:55:46,614 WARN [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 78f10d0299f2958c87d34c58cfcb3253 to self. 2024-11-25T10:55:46,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=CLOSED 2024-11-25T10:55:46,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:55:46,624 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:46,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T10:55:46,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 in 201 msec 2024-11-25T10:55:46,627 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, REOPEN/MOVE; state=CLOSED, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=true 2024-11-25T10:55:46,781 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:46,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, REOPEN/MOVE because future has completed 2024-11-25T10:55:46,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:55:46,978 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:46,978 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:55:46,979 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:46,979 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:55:46,979 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:46,979 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:46,993 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:46,997 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:47,004 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78f10d0299f2958c87d34c58cfcb3253 columnFamilyName A 2024-11-25T10:55:47,015 DEBUG [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:47,016 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(327): Store=78f10d0299f2958c87d34c58cfcb3253/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:47,016 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,017 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:47,018 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78f10d0299f2958c87d34c58cfcb3253 columnFamilyName B 2024-11-25T10:55:47,018 DEBUG [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:47,020 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(327): Store=78f10d0299f2958c87d34c58cfcb3253/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:47,020 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,028 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:55:47,029 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78f10d0299f2958c87d34c58cfcb3253 columnFamilyName C 2024-11-25T10:55:47,029 DEBUG [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:47,032 INFO [StoreOpener-78f10d0299f2958c87d34c58cfcb3253-1 {}] regionserver.HStore(327): Store=78f10d0299f2958c87d34c58cfcb3253/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:55:47,034 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,034 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,035 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,047 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,059 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,059 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,062 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:55:47,071 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,075 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 78f10d0299f2958c87d34c58cfcb3253; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60102884, jitterRate=-0.1043972373008728}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:55:47,075 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,077 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 78f10d0299f2958c87d34c58cfcb3253: Running coprocessor pre-open hook at 1732532146980Writing region info on filesystem at 1732532146980Initializing all the Stores at 1732532146985 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532146985Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532146986 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532146986Cleaning up temporary data from old regions at 1732532147059 (+73 ms)Running coprocessor post-open hooks at 1732532147075 (+16 ms)Region opened successfully at 1732532147077 (+2 ms) 2024-11-25T10:55:47,084 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., pid=33, masterSystemTime=1732532146964 2024-11-25T10:55:47,086 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,086 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,087 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=OPEN, openSeqNum=5, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:55:47,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-25T10:55:47,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 in 282 msec 2024-11-25T10:55:47,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-25T10:55:47,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, REOPEN/MOVE in 683 msec 2024-11-25T10:55:47,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T10:55:47,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 727 msec 2024-11-25T10:55:47,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-25T10:55:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T10:55:47,151 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:47,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323fc298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,155 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c06acd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ca4672a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,163 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488d30ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,169 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea63494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,174 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df792e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f0e2b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,185 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e20c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,191 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c04fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,200 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8cf03a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,203 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273cb125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,208 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487d58e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,211 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4079fea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,216 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f3237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634b6b54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,222 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641c9214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,225 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ae3515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:55:47,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:55:47,237 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:55:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:55:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:55:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7769568e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:55:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:55:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:55:47,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:55:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ed4814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:55:47,269 DEBUG [Thread-546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,271 DEBUG [Thread-544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,275 DEBUG [Thread-542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,286 DEBUG [Thread-548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,288 DEBUG [Thread-551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,289 DEBUG [Thread-551 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,290 DEBUG [Thread-551 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:55:47,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:47,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:47,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:47,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:47,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:47,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:47,326 DEBUG [Thread-553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,327 DEBUG [Thread-555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,333 DEBUG [Thread-557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,335 DEBUG [Thread-555 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,336 DEBUG [Thread-557 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,336 DEBUG [Thread-557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,341 DEBUG [Thread-557 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,349 DEBUG [Thread-555 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,352 DEBUG [Thread-540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:55:47,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,354 DEBUG [Thread-540 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:55:47,354 DEBUG [Thread-540 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:55:47,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:55:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:55:47,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532207379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532207384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532207386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532207390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-25T10:55:47,416 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:47,417 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:47,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T10:55:47,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532207403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f0a77b4e502b41c085c0f57c8dfc5b79_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_1/A:col10/1732532147297/Put/seqid=0 2024-11-25T10:55:47,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741927_1103 (size=14594) 2024-11-25T10:55:47,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:47,528 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f0a77b4e502b41c085c0f57c8dfc5b79_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f0a77b4e502b41c085c0f57c8dfc5b79_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T10:55:47,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/02548c4803f64acfadbeba509d7fc131, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532207518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532207518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532207518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532207518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/02548c4803f64acfadbeba509d7fc131 is 175, key is test_row_1/A:col10/1732532147297/Put/seqid=0 2024-11-25T10:55:47,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532207538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-25T10:55:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741928_1104 (size=39545) 2024-11-25T10:55:47,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/02548c4803f64acfadbeba509d7fc131 2024-11-25T10:55:47,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/ea69b18f4c0c4508b3af4e5b301b6fc4 is 50, key is test_row_1/B:col10/1732532147297/Put/seqid=0 2024-11-25T10:55:47,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741929_1105 (size=9657) 2024-11-25T10:55:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T10:55:47,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-25T10:55:47,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:47,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532207737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532207749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532207749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532207752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532207761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:47,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:47,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-25T10:55:47,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:47,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:47,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T10:55:48,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-25T10:55:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:48,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532208070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532208073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532208071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532208073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532208083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/ea69b18f4c0c4508b3af4e5b301b6fc4 2024-11-25T10:55:48,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-25T10:55:48,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:48,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:48,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:48,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/22404d4fa2b84233a4935f19cc4e675b is 50, key is test_row_1/C:col10/1732532147297/Put/seqid=0 2024-11-25T10:55:48,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741930_1106 (size=9657) 2024-11-25T10:55:48,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/22404d4fa2b84233a4935f19cc4e675b 2024-11-25T10:55:48,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/02548c4803f64acfadbeba509d7fc131 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/02548c4803f64acfadbeba509d7fc131 2024-11-25T10:55:48,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/02548c4803f64acfadbeba509d7fc131, entries=200, sequenceid=16, filesize=38.6 K 2024-11-25T10:55:48,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/ea69b18f4c0c4508b3af4e5b301b6fc4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea69b18f4c0c4508b3af4e5b301b6fc4 2024-11-25T10:55:48,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-25T10:55:48,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea69b18f4c0c4508b3af4e5b301b6fc4, entries=100, sequenceid=16, filesize=9.4 K 2024-11-25T10:55:48,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:48,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:48,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:48,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/22404d4fa2b84233a4935f19cc4e675b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/22404d4fa2b84233a4935f19cc4e675b 2024-11-25T10:55:48,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/22404d4fa2b84233a4935f19cc4e675b, entries=100, sequenceid=16, filesize=9.4 K 2024-11-25T10:55:48,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 78f10d0299f2958c87d34c58cfcb3253 in 1102ms, sequenceid=16, compaction requested=false 2024-11-25T10:55:48,411 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T10:55:48,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:48,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-25T10:55:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:48,526 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T10:55:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:48,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T10:55:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b4c9abd53bc5430c8250bde72647ceca_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532147363/Put/seqid=0 2024-11-25T10:55:48,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:48,657 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T10:55:48,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741931_1107 (size=12154) 2024-11-25T10:55:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:48,677 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b4c9abd53bc5430c8250bde72647ceca_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b4c9abd53bc5430c8250bde72647ceca_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:48,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/7a759055b742413587da2483cc1089bf, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:48,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/7a759055b742413587da2483cc1089bf is 175, key is test_row_0/A:col10/1732532147363/Put/seqid=0 2024-11-25T10:55:48,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532208650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532208644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532208659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532208690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532208693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741932_1108 (size=30955) 2024-11-25T10:55:48,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532208804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532208805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532208815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532208834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532208837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532209024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532209027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532209034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532209063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532209063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,136 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/7a759055b742413587da2483cc1089bf 2024-11-25T10:55:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/d7f4ee56e7f641afa5a5fae66837a932 is 50, key is test_row_0/B:col10/1732532147363/Put/seqid=0 2024-11-25T10:55:49,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741933_1109 (size=12001) 2024-11-25T10:55:49,232 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/d7f4ee56e7f641afa5a5fae66837a932 2024-11-25T10:55:49,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0a4636b762734b4aa5b033771dae369c is 50, key is test_row_0/C:col10/1732532147363/Put/seqid=0 2024-11-25T10:55:49,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532209344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532209344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741934_1110 (size=12001) 2024-11-25T10:55:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532209353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532209374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532209385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T10:55:49,765 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0a4636b762734b4aa5b033771dae369c 2024-11-25T10:55:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/7a759055b742413587da2483cc1089bf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/7a759055b742413587da2483cc1089bf 2024-11-25T10:55:49,782 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/7a759055b742413587da2483cc1089bf, entries=150, sequenceid=36, filesize=30.2 K 2024-11-25T10:55:49,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/d7f4ee56e7f641afa5a5fae66837a932 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/d7f4ee56e7f641afa5a5fae66837a932 2024-11-25T10:55:49,794 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/d7f4ee56e7f641afa5a5fae66837a932, entries=150, sequenceid=36, filesize=11.7 K 2024-11-25T10:55:49,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0a4636b762734b4aa5b033771dae369c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0a4636b762734b4aa5b033771dae369c 2024-11-25T10:55:49,806 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0a4636b762734b4aa5b033771dae369c, entries=150, sequenceid=36, filesize=11.7 K 2024-11-25T10:55:49,809 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 78f10d0299f2958c87d34c58cfcb3253 in 1281ms, sequenceid=36, compaction requested=false 2024-11-25T10:55:49,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:49,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:49,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-25T10:55:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-25T10:55:49,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-25T10:55:49,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3940 sec 2024-11-25T10:55:49,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.4020 sec 2024-11-25T10:55:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:49,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:55:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:49,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fb54397504b646329b7935bfda0bc202_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532149865/Put/seqid=0 2024-11-25T10:55:49,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741935_1111 (size=14594) 2024-11-25T10:55:49,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:49,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fb54397504b646329b7935bfda0bc202_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125fb54397504b646329b7935bfda0bc202_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:49,973 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a17f730745074ffea936781ccb248470, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:49,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a17f730745074ffea936781ccb248470 is 175, key is test_row_0/A:col10/1732532149865/Put/seqid=0 2024-11-25T10:55:49,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532209946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532209949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532209963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532209981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532209982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:49,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741936_1112 (size=39549) 2024-11-25T10:55:50,030 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a17f730745074ffea936781ccb248470 2024-11-25T10:55:50,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/e400c9c2aad947688194b40ae7356eff is 50, key is test_row_0/B:col10/1732532149865/Put/seqid=0 2024-11-25T10:55:50,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532210085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532210094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532210096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532210094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532210109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741937_1113 (size=12001) 2024-11-25T10:55:50,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/e400c9c2aad947688194b40ae7356eff 2024-11-25T10:55:50,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/6339dda1cda547b5894bfa4e5d825270 is 50, key is test_row_0/C:col10/1732532149865/Put/seqid=0 2024-11-25T10:55:50,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741938_1114 (size=12001) 2024-11-25T10:55:50,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/6339dda1cda547b5894bfa4e5d825270 2024-11-25T10:55:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a17f730745074ffea936781ccb248470 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a17f730745074ffea936781ccb248470 2024-11-25T10:55:50,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a17f730745074ffea936781ccb248470, entries=200, sequenceid=49, filesize=38.6 K 2024-11-25T10:55:50,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/e400c9c2aad947688194b40ae7356eff as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/e400c9c2aad947688194b40ae7356eff 2024-11-25T10:55:50,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/e400c9c2aad947688194b40ae7356eff, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T10:55:50,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/6339dda1cda547b5894bfa4e5d825270 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/6339dda1cda547b5894bfa4e5d825270 2024-11-25T10:55:50,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532210304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532210305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532210314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532210314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532210324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/6339dda1cda547b5894bfa4e5d825270, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T10:55:50,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 78f10d0299f2958c87d34c58cfcb3253 in 478ms, sequenceid=49, compaction requested=true 2024-11-25T10:55:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:50,349 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:50,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:50,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:50,353 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:50,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:50,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:50,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:50,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:50,354 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:50,354 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/A is initiating minor compaction (all files) 2024-11-25T10:55:50,354 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/A in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:50,354 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/02548c4803f64acfadbeba509d7fc131, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/7a759055b742413587da2483cc1089bf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a17f730745074ffea936781ccb248470] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=107.5 K 2024-11-25T10:55:50,354 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:50,355 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/02548c4803f64acfadbeba509d7fc131, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/7a759055b742413587da2483cc1089bf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a17f730745074ffea936781ccb248470] 2024-11-25T10:55:50,363 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02548c4803f64acfadbeba509d7fc131, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732532147297 2024-11-25T10:55:50,365 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a759055b742413587da2483cc1089bf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732532147363 2024-11-25T10:55:50,368 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:50,369 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/B is initiating minor compaction (all files) 2024-11-25T10:55:50,369 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/B in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:50,369 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea69b18f4c0c4508b3af4e5b301b6fc4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/d7f4ee56e7f641afa5a5fae66837a932, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/e400c9c2aad947688194b40ae7356eff] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=32.9 K 2024-11-25T10:55:50,369 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17f730745074ffea936781ccb248470, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532148635 2024-11-25T10:55:50,369 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ea69b18f4c0c4508b3af4e5b301b6fc4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732532147297 2024-11-25T10:55:50,370 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f4ee56e7f641afa5a5fae66837a932, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732532147363 2024-11-25T10:55:50,372 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e400c9c2aad947688194b40ae7356eff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532148661 2024-11-25T10:55:50,425 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:50,437 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:50,438 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/cb6cd7f4b83e41159e5bd6e55278bffd is 50, key is test_row_0/B:col10/1732532149865/Put/seqid=0 2024-11-25T10:55:50,445 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251f73acf160f74639942d7eff66ab2163_78f10d0299f2958c87d34c58cfcb3253 store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:50,462 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251f73acf160f74639942d7eff66ab2163_78f10d0299f2958c87d34c58cfcb3253, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:50,462 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251f73acf160f74639942d7eff66ab2163_78f10d0299f2958c87d34c58cfcb3253 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:50,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741939_1115 (size=12104) 2024-11-25T10:55:50,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741940_1116 (size=4469) 2024-11-25T10:55:50,517 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#A#compaction#96 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:50,517 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/b1aed88ff229401eab698225164fb664 is 175, key is test_row_0/A:col10/1732532149865/Put/seqid=0 2024-11-25T10:55:50,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741941_1117 (size=31058) 2024-11-25T10:55:50,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532149946, max timestamp=1732532150643Num uniques 859; 2024-11-25T10:55:50,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:55:50,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=84.98 KB heapSize=247.03 KB 2024-11-25T10:55:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:50,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732532149946, max timestamp=1732532150643Num uniques 859; 2024-11-25T10:55:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:55:50,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:55:50,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125687428b784c540f8b291fa3608a5efbc_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532150643/Put/seqid=0 2024-11-25T10:55:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532210735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741942_1118 (size=17034) 2024-11-25T10:55:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532210746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532210735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,785 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532210765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125687428b784c540f8b291fa3608a5efbc_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125687428b784c540f8b291fa3608a5efbc_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:50,798 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/1975b70068e24dea8b9d02e0807d89cd, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:50,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/1975b70068e24dea8b9d02e0807d89cd is 175, key is test_row_0/A:col10/1732532150643/Put/seqid=0 2024-11-25T10:55:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532210784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741943_1119 (size=48139) 2024-11-25T10:55:50,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/1975b70068e24dea8b9d02e0807d89cd 2024-11-25T10:55:50,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532210894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532210894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532210898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532210904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2e657c464d0349359380d419d7f1a47e is 50, key is test_row_0/B:col10/1732532150643/Put/seqid=0 2024-11-25T10:55:50,915 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/cb6cd7f4b83e41159e5bd6e55278bffd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cb6cd7f4b83e41159e5bd6e55278bffd 2024-11-25T10:55:50,923 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/B of 78f10d0299f2958c87d34c58cfcb3253 into cb6cd7f4b83e41159e5bd6e55278bffd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:50,923 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:50,923 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/B, priority=13, startTime=1732532150353; duration=0sec 2024-11-25T10:55:50,923 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:50,923 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:B 2024-11-25T10:55:50,923 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:55:50,925 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:55:50,925 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/C is initiating minor compaction (all files) 2024-11-25T10:55:50,925 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/C in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:50,925 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/22404d4fa2b84233a4935f19cc4e675b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0a4636b762734b4aa5b033771dae369c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/6339dda1cda547b5894bfa4e5d825270] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=32.9 K 2024-11-25T10:55:50,926 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 22404d4fa2b84233a4935f19cc4e675b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732532147297 2024-11-25T10:55:50,926 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4636b762734b4aa5b033771dae369c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732532147363 2024-11-25T10:55:50,927 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6339dda1cda547b5894bfa4e5d825270, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532148661 2024-11-25T10:55:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532210915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:50,951 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#C#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:50,952 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/ab5cd94e4fe14cfc9b5f44c7bd7d1ac3 is 50, key is test_row_0/C:col10/1732532149865/Put/seqid=0 2024-11-25T10:55:50,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741944_1120 (size=12001) 2024-11-25T10:55:50,998 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/b1aed88ff229401eab698225164fb664 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b1aed88ff229401eab698225164fb664 2024-11-25T10:55:51,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741945_1121 (size=12104) 2024-11-25T10:55:51,014 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/A of 78f10d0299f2958c87d34c58cfcb3253 into b1aed88ff229401eab698225164fb664(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:51,014 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:51,014 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/A, priority=13, startTime=1732532150349; duration=0sec 2024-11-25T10:55:51,014 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:51,014 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:A 2024-11-25T10:55:51,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532211115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532211115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532211118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532211118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532211136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2e657c464d0349359380d419d7f1a47e 2024-11-25T10:55:51,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/986ffbd1dace4daba7d6d1d6ba8791a8 is 50, key is test_row_0/C:col10/1732532150643/Put/seqid=0 2024-11-25T10:55:51,421 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/ab5cd94e4fe14cfc9b5f44c7bd7d1ac3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ab5cd94e4fe14cfc9b5f44c7bd7d1ac3 2024-11-25T10:55:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532211424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532211424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532211424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,435 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/C of 78f10d0299f2958c87d34c58cfcb3253 into ab5cd94e4fe14cfc9b5f44c7bd7d1ac3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:51,435 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:51,435 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/C, priority=13, startTime=1732532150354; duration=0sec 2024-11-25T10:55:51,435 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:51,435 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:C 2024-11-25T10:55:51,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532211433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532211444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741946_1122 (size=12001) 2024-11-25T10:55:51,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/986ffbd1dace4daba7d6d1d6ba8791a8 2024-11-25T10:55:51,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/1975b70068e24dea8b9d02e0807d89cd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/1975b70068e24dea8b9d02e0807d89cd 2024-11-25T10:55:51,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/1975b70068e24dea8b9d02e0807d89cd, entries=250, sequenceid=70, filesize=47.0 K 2024-11-25T10:55:51,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2e657c464d0349359380d419d7f1a47e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2e657c464d0349359380d419d7f1a47e 2024-11-25T10:55:51,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2e657c464d0349359380d419d7f1a47e, entries=150, sequenceid=70, filesize=11.7 K 2024-11-25T10:55:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/986ffbd1dace4daba7d6d1d6ba8791a8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/986ffbd1dace4daba7d6d1d6ba8791a8 2024-11-25T10:55:51,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/986ffbd1dace4daba7d6d1d6ba8791a8, entries=150, sequenceid=70, filesize=11.7 K 2024-11-25T10:55:51,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.62 KB/64120, heapSize ~174.28 KB/178464, currentSize=114.05 KB/116790 for 78f10d0299f2958c87d34c58cfcb3253 in 874ms, sequenceid=70, compaction requested=false 2024-11-25T10:55:51,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T10:55:51,606 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:51,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-25T10:55:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T10:55:51,612 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:51,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:51,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T10:55:51,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-25T10:55:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:51,805 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T10:55:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:51,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bdb5f5672fb043ecb0c9867aa0c3b8b6_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532150752/Put/seqid=0 2024-11-25T10:55:51,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741947_1123 (size=12154) 2024-11-25T10:55:51,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T10:55:51,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532211965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532211976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532211982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532211982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532211982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532212094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532212094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532212094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532212103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T10:55:52,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:52,314 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bdb5f5672fb043ecb0c9867aa0c3b8b6_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bdb5f5672fb043ecb0c9867aa0c3b8b6_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a43d6261841e4c2cbb162f1dfe2ab577, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a43d6261841e4c2cbb162f1dfe2ab577 is 175, key is test_row_0/A:col10/1732532150752/Put/seqid=0 2024-11-25T10:55:52,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532212319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532212319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532212320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532212321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741948_1124 (size=30955) 2024-11-25T10:55:52,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532212634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532212644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532212646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532212647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:52,759 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a43d6261841e4c2cbb162f1dfe2ab577 2024-11-25T10:55:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T10:55:52,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/b3c0865bab4644b0baafc62fa809cbf8 is 50, key is test_row_0/B:col10/1732532150752/Put/seqid=0 2024-11-25T10:55:52,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741949_1125 (size=12001) 2024-11-25T10:55:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532212995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:52,998 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in 78f10d0299f2958c87d34c58cfcb3253 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:55:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532213148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532213165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532213165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532213165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:53,266 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/b3c0865bab4644b0baafc62fa809cbf8 2024-11-25T10:55:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/bf2506933c3f47159c0d0acb38927473 is 50, key is test_row_0/C:col10/1732532150752/Put/seqid=0 2024-11-25T10:55:53,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741950_1126 (size=12001) 2024-11-25T10:55:53,367 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/bf2506933c3f47159c0d0acb38927473 2024-11-25T10:55:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a43d6261841e4c2cbb162f1dfe2ab577 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a43d6261841e4c2cbb162f1dfe2ab577 2024-11-25T10:55:53,428 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a43d6261841e4c2cbb162f1dfe2ab577, entries=150, sequenceid=93, filesize=30.2 K 2024-11-25T10:55:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/b3c0865bab4644b0baafc62fa809cbf8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/b3c0865bab4644b0baafc62fa809cbf8 2024-11-25T10:55:53,474 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/b3c0865bab4644b0baafc62fa809cbf8, entries=150, sequenceid=93, filesize=11.7 K 2024-11-25T10:55:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/bf2506933c3f47159c0d0acb38927473 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/bf2506933c3f47159c0d0acb38927473 2024-11-25T10:55:53,501 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/bf2506933c3f47159c0d0acb38927473, entries=150, sequenceid=93, filesize=11.7 K 2024-11-25T10:55:53,509 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 78f10d0299f2958c87d34c58cfcb3253 in 1704ms, sequenceid=93, compaction requested=true 2024-11-25T10:55:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-25T10:55:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-25T10:55:53,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-25T10:55:53,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-11-25T10:55:53,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.9150 sec 2024-11-25T10:55:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T10:55:53,797 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:53,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-25T10:55:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T10:55:53,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:53,802 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:53,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T10:55:53,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T10:55:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:53,970 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:55:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:54,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259372a7bd6351481ea92f8c31b6380048_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532151979/Put/seqid=0 2024-11-25T10:55:54,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741951_1127 (size=12154) 2024-11-25T10:55:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:54,096 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259372a7bd6351481ea92f8c31b6380048_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259372a7bd6351481ea92f8c31b6380048_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/c65967c541234556be995b7e9e2b8b1f, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/c65967c541234556be995b7e9e2b8b1f is 175, key is test_row_0/A:col10/1732532151979/Put/seqid=0 2024-11-25T10:55:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T10:55:54,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741952_1128 (size=30955) 2024-11-25T10:55:54,169 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/c65967c541234556be995b7e9e2b8b1f 2024-11-25T10:55:54,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/5f8a3ecfbda042c4a87d9ba0182f2c2b is 50, key is test_row_0/B:col10/1732532151979/Put/seqid=0 2024-11-25T10:55:54,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741953_1129 (size=12001) 2024-11-25T10:55:54,271 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/5f8a3ecfbda042c4a87d9ba0182f2c2b 2024-11-25T10:55:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532214270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532214273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532214285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532214297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/4dd3138a8b7a40debddbdbaa25f7faef is 50, key is test_row_0/C:col10/1732532151979/Put/seqid=0 2024-11-25T10:55:54,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741954_1130 (size=12001) 2024-11-25T10:55:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532214405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532214406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532214406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532214414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T10:55:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532214625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532214625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532214628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532214628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:54,780 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/4dd3138a8b7a40debddbdbaa25f7faef 2024-11-25T10:55:54,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/c65967c541234556be995b7e9e2b8b1f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/c65967c541234556be995b7e9e2b8b1f 2024-11-25T10:55:54,807 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/c65967c541234556be995b7e9e2b8b1f, entries=150, sequenceid=106, filesize=30.2 K 2024-11-25T10:55:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/5f8a3ecfbda042c4a87d9ba0182f2c2b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5f8a3ecfbda042c4a87d9ba0182f2c2b 2024-11-25T10:55:54,842 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5f8a3ecfbda042c4a87d9ba0182f2c2b, entries=150, sequenceid=106, filesize=11.7 K 2024-11-25T10:55:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/4dd3138a8b7a40debddbdbaa25f7faef as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4dd3138a8b7a40debddbdbaa25f7faef 2024-11-25T10:55:54,871 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4dd3138a8b7a40debddbdbaa25f7faef, entries=150, sequenceid=106, filesize=11.7 K 2024-11-25T10:55:54,873 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 78f10d0299f2958c87d34c58cfcb3253 in 904ms, sequenceid=106, compaction requested=true 2024-11-25T10:55:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-25T10:55:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-25T10:55:54,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-25T10:55:54,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-11-25T10:55:54,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.0830 sec 2024-11-25T10:55:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T10:55:54,938 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:54,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-25T10:55:54,942 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:54,945 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:54,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T10:55:54,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T10:55:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:55,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257dc900fe59f04bb1b976277b586f97a1_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532154270/Put/seqid=0 2024-11-25T10:55:55,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532215029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532215034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T10:55:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532215044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532215048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532215051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741955_1131 (size=12154) 2024-11-25T10:55:55,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:55,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257dc900fe59f04bb1b976277b586f97a1_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257dc900fe59f04bb1b976277b586f97a1_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:55,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/014e8dfc29a04efe8da98dfcd6585d81, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:55,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/014e8dfc29a04efe8da98dfcd6585d81 is 175, key is test_row_0/A:col10/1732532154270/Put/seqid=0 2024-11-25T10:55:55,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:55,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:55,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741956_1132 (size=30955) 2024-11-25T10:55:55,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=126, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/014e8dfc29a04efe8da98dfcd6585d81 2024-11-25T10:55:55,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532215154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532215154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532215164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532215164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2066118267354f87806f56786b53092e is 50, key is test_row_0/B:col10/1732532154270/Put/seqid=0 2024-11-25T10:55:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532215174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741957_1133 (size=12001) 2024-11-25T10:55:55,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2066118267354f87806f56786b53092e 2024-11-25T10:55:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T10:55:55,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/96ae99068aca454f854ccde6ebf88c20 is 50, key is test_row_0/C:col10/1732532154270/Put/seqid=0 2024-11-25T10:55:55,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741958_1134 (size=12001) 2024-11-25T10:55:55,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/96ae99068aca454f854ccde6ebf88c20 2024-11-25T10:55:55,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/014e8dfc29a04efe8da98dfcd6585d81 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/014e8dfc29a04efe8da98dfcd6585d81 2024-11-25T10:55:55,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/014e8dfc29a04efe8da98dfcd6585d81, entries=150, sequenceid=126, filesize=30.2 K 2024-11-25T10:55:55,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2066118267354f87806f56786b53092e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2066118267354f87806f56786b53092e 2024-11-25T10:55:55,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2066118267354f87806f56786b53092e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-25T10:55:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/96ae99068aca454f854ccde6ebf88c20 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/96ae99068aca454f854ccde6ebf88c20 2024-11-25T10:55:55,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/96ae99068aca454f854ccde6ebf88c20, entries=150, sequenceid=126, filesize=11.7 K 2024-11-25T10:55:55,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 78f10d0299f2958c87d34c58cfcb3253 in 418ms, sequenceid=126, compaction requested=true 2024-11-25T10:55:55,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:55,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:55:55,396 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T10:55:55,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:55,396 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T10:55:55,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:55:55,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:55,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:55:55,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:55,400 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172062 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T10:55:55,400 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/A is initiating minor compaction (all files) 2024-11-25T10:55:55,400 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/A in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,400 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b1aed88ff229401eab698225164fb664, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/1975b70068e24dea8b9d02e0807d89cd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a43d6261841e4c2cbb162f1dfe2ab577, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/c65967c541234556be995b7e9e2b8b1f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/014e8dfc29a04efe8da98dfcd6585d81] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=168.0 K 2024-11-25T10:55:55,400 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,400 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b1aed88ff229401eab698225164fb664, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/1975b70068e24dea8b9d02e0807d89cd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a43d6261841e4c2cbb162f1dfe2ab577, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/c65967c541234556be995b7e9e2b8b1f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/014e8dfc29a04efe8da98dfcd6585d81] 2024-11-25T10:55:55,401 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T10:55:55,401 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/B is initiating minor compaction (all files) 2024-11-25T10:55:55,401 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/B in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,401 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cb6cd7f4b83e41159e5bd6e55278bffd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2e657c464d0349359380d419d7f1a47e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/b3c0865bab4644b0baafc62fa809cbf8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5f8a3ecfbda042c4a87d9ba0182f2c2b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2066118267354f87806f56786b53092e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=58.7 K 2024-11-25T10:55:55,402 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting cb6cd7f4b83e41159e5bd6e55278bffd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532148661 2024-11-25T10:55:55,402 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1aed88ff229401eab698225164fb664, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532148661 2024-11-25T10:55:55,402 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e657c464d0349359380d419d7f1a47e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732532149946 2024-11-25T10:55:55,403 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1975b70068e24dea8b9d02e0807d89cd, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732532149946 2024-11-25T10:55:55,403 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting b3c0865bab4644b0baafc62fa809cbf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732532150699 2024-11-25T10:55:55,403 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a43d6261841e4c2cbb162f1dfe2ab577, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732532150699 2024-11-25T10:55:55,404 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8a3ecfbda042c4a87d9ba0182f2c2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732532151968 2024-11-25T10:55:55,404 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65967c541234556be995b7e9e2b8b1f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732532151968 2024-11-25T10:55:55,405 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2066118267354f87806f56786b53092e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732532154270 2024-11-25T10:55:55,405 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014e8dfc29a04efe8da98dfcd6585d81, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732532154270 2024-11-25T10:55:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:55,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T10:55:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:55,436 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#B#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:55,437 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/ea4b6f8af55e4fe484257bcef2489b8e is 50, key is test_row_0/B:col10/1732532154270/Put/seqid=0 2024-11-25T10:55:55,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:55,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,477 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:55,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e174abfb9d52451da35ca3d1f0adeb56_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532155414/Put/seqid=0 2024-11-25T10:55:55,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741959_1135 (size=12275) 2024-11-25T10:55:55,514 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411252e570c877931440ca7e5d7821cde4508_78f10d0299f2958c87d34c58cfcb3253 store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:55,518 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411252e570c877931440ca7e5d7821cde4508_78f10d0299f2958c87d34c58cfcb3253, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:55,519 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252e570c877931440ca7e5d7821cde4508_78f10d0299f2958c87d34c58cfcb3253 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:55,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532215493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532215504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532215528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532215533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532215533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741960_1136 (size=12304) 2024-11-25T10:55:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T10:55:55,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:55,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:55,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741961_1137 (size=4469) 2024-11-25T10:55:55,638 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#A#compaction#112 average throughput is 0.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:55,639 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/43e6a61ef62844cba7f36bea5461fbbe is 175, key is test_row_0/A:col10/1732532154270/Put/seqid=0 2024-11-25T10:55:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532215644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532215656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532215675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532215676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532215676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741962_1138 (size=31229) 2024-11-25T10:55:55,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532215857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532215886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532215888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532215889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532215889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:55,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:55,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:55,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,945 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/ea4b6f8af55e4fe484257bcef2489b8e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea4b6f8af55e4fe484257bcef2489b8e 2024-11-25T10:55:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:55,968 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/B of 78f10d0299f2958c87d34c58cfcb3253 into ea4b6f8af55e4fe484257bcef2489b8e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:55,968 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:55,968 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/B, priority=11, startTime=1732532155396; duration=0sec 2024-11-25T10:55:55,968 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:55:55,968 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:B 2024-11-25T10:55:55,973 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T10:55:55,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:55,999 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T10:55:55,999 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/C is initiating minor compaction (all files) 2024-11-25T10:55:55,999 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/C in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:55,999 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ab5cd94e4fe14cfc9b5f44c7bd7d1ac3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/986ffbd1dace4daba7d6d1d6ba8791a8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/bf2506933c3f47159c0d0acb38927473, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4dd3138a8b7a40debddbdbaa25f7faef, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/96ae99068aca454f854ccde6ebf88c20] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=58.7 K 2024-11-25T10:55:56,001 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5cd94e4fe14cfc9b5f44c7bd7d1ac3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532148661 2024-11-25T10:55:56,002 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 986ffbd1dace4daba7d6d1d6ba8791a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732532149946 2024-11-25T10:55:56,003 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2506933c3f47159c0d0acb38927473, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732532150699 2024-11-25T10:55:56,005 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd3138a8b7a40debddbdbaa25f7faef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732532151968 2024-11-25T10:55:56,006 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ae99068aca454f854ccde6ebf88c20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732532154270 2024-11-25T10:55:56,007 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e174abfb9d52451da35ca3d1f0adeb56_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e174abfb9d52451da35ca3d1f0adeb56_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:56,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/e5a572256c224022836ce3073c71b0a3, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:56,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/e5a572256c224022836ce3073c71b0a3 is 175, key is test_row_0/A:col10/1732532155414/Put/seqid=0 2024-11-25T10:55:56,028 INFO [master/6bfc37bad115:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T10:55:56,028 INFO [master/6bfc37bad115:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T10:55:56,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741963_1139 (size=31105) 2024-11-25T10:55:56,082 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#C#compaction#114 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:55:56,082 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/aaa0420d1f2d49129ce49d5698ede242 is 50, key is test_row_0/C:col10/1732532154270/Put/seqid=0 2024-11-25T10:55:56,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:56,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:56,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T10:55:56,122 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/43e6a61ef62844cba7f36bea5461fbbe as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/43e6a61ef62844cba7f36bea5461fbbe 2024-11-25T10:55:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741964_1140 (size=12275) 2024-11-25T10:55:56,140 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/A of 78f10d0299f2958c87d34c58cfcb3253 into 43e6a61ef62844cba7f36bea5461fbbe(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:56,140 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:56,140 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/A, priority=11, startTime=1732532155395; duration=0sec 2024-11-25T10:55:56,141 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:56,141 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:A 2024-11-25T10:55:56,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532216183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532216225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532216229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532216229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532216230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/e5a572256c224022836ce3073c71b0a3 2024-11-25T10:55:56,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/8ad99027aa644846b8a1eee42692f1a2 is 50, key is test_row_0/B:col10/1732532155414/Put/seqid=0 2024-11-25T10:55:56,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741965_1141 (size=12151) 2024-11-25T10:55:56,549 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/aaa0420d1f2d49129ce49d5698ede242 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/aaa0420d1f2d49129ce49d5698ede242 2024-11-25T10:55:56,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,588 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/C of 78f10d0299f2958c87d34c58cfcb3253 into aaa0420d1f2d49129ce49d5698ede242(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:55:56,588 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:56,588 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/C, priority=11, startTime=1732532155396; duration=0sec 2024-11-25T10:55:56,588 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:55:56,588 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:C 2024-11-25T10:55:56,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532216708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:56,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532216749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532216753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532216754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532216754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:56,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:56,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:56,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:56,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:56,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/8ad99027aa644846b8a1eee42692f1a2 2024-11-25T10:55:56,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/04937fb736434b6eb22b7a2d503ce469 is 50, key is test_row_0/C:col10/1732532155414/Put/seqid=0 2024-11-25T10:55:57,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741966_1142 (size=12151) 2024-11-25T10:55:57,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/04937fb736434b6eb22b7a2d503ce469 2024-11-25T10:55:57,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:57,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:57,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/e5a572256c224022836ce3073c71b0a3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/e5a572256c224022836ce3073c71b0a3 2024-11-25T10:55:57,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/e5a572256c224022836ce3073c71b0a3, entries=150, sequenceid=142, filesize=30.4 K 2024-11-25T10:55:57,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/8ad99027aa644846b8a1eee42692f1a2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/8ad99027aa644846b8a1eee42692f1a2 2024-11-25T10:55:57,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/8ad99027aa644846b8a1eee42692f1a2, entries=150, sequenceid=142, filesize=11.9 K 2024-11-25T10:55:57,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/04937fb736434b6eb22b7a2d503ce469 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/04937fb736434b6eb22b7a2d503ce469 2024-11-25T10:55:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T10:55:57,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/04937fb736434b6eb22b7a2d503ce469, entries=150, sequenceid=142, filesize=11.9 K 2024-11-25T10:55:57,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 78f10d0299f2958c87d34c58cfcb3253 in 1730ms, sequenceid=142, compaction requested=false 2024-11-25T10:55:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:57,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T10:55:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:57,222 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:55:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:57,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f42e35bd0b68469eac129ed1190aa9b5_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532155463/Put/seqid=0 2024-11-25T10:55:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741967_1143 (size=12304) 2024-11-25T10:55:57,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:57,733 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f42e35bd0b68469eac129ed1190aa9b5_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f42e35bd0b68469eac129ed1190aa9b5_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:57,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/8a43fe3ed22a47c9a79273c0c55d4624, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:57,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/8a43fe3ed22a47c9a79273c0c55d4624 is 175, key is test_row_0/A:col10/1732532155463/Put/seqid=0 2024-11-25T10:55:57,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741968_1144 (size=31105) 2024-11-25T10:55:57,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532217805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532217809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532217830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532217833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532217840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532217943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532217945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532217958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532217965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:57,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532217965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532218157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532218161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532218177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532218185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532218185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,217 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/8a43fe3ed22a47c9a79273c0c55d4624 2024-11-25T10:55:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/1b091ef19fdd47e5bc5ae712c7465fcd is 50, key is test_row_0/B:col10/1732532155463/Put/seqid=0 2024-11-25T10:55:58,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741969_1145 (size=12151) 2024-11-25T10:55:58,307 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/1b091ef19fdd47e5bc5ae712c7465fcd 2024-11-25T10:55:58,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/a78af616eaef4545b9e60d7b627ee571 is 50, key is test_row_0/C:col10/1732532155463/Put/seqid=0 2024-11-25T10:55:58,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741970_1146 (size=12151) 2024-11-25T10:55:58,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532218474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532218474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532218493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532218505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532218505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:58,816 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/a78af616eaef4545b9e60d7b627ee571 2024-11-25T10:55:58,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/8a43fe3ed22a47c9a79273c0c55d4624 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/8a43fe3ed22a47c9a79273c0c55d4624 2024-11-25T10:55:58,861 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/8a43fe3ed22a47c9a79273c0c55d4624, entries=150, sequenceid=162, filesize=30.4 K 2024-11-25T10:55:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/1b091ef19fdd47e5bc5ae712c7465fcd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/1b091ef19fdd47e5bc5ae712c7465fcd 2024-11-25T10:55:58,934 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/1b091ef19fdd47e5bc5ae712c7465fcd, entries=150, sequenceid=162, filesize=11.9 K 2024-11-25T10:55:58,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/a78af616eaef4545b9e60d7b627ee571 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/a78af616eaef4545b9e60d7b627ee571 2024-11-25T10:55:58,970 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/a78af616eaef4545b9e60d7b627ee571, entries=150, sequenceid=162, filesize=11.9 K 2024-11-25T10:55:58,977 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 78f10d0299f2958c87d34c58cfcb3253 in 1754ms, sequenceid=162, compaction requested=true 2024-11-25T10:55:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:55:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:58,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-25T10:55:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-25T10:55:58,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-25T10:55:58,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0340 sec 2024-11-25T10:55:58,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 4.0430 sec 2024-11-25T10:55:59,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:55:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:59,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:55:59,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:59,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:55:59,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:59,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:55:59,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:55:59,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125066492813b5e477fb4614920c95a7d07_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532159003/Put/seqid=0 2024-11-25T10:55:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532219087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532219097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532219100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532219103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532219103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741971_1147 (size=14794) 2024-11-25T10:55:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T10:55:59,134 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:55:59,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:55:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-25T10:55:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T10:55:59,137 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:55:59,139 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:55:59,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:55:59,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532219217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532219217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532219217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532219217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532219217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T10:55:59,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:55:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532219433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532219434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532219435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532219435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532219435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:55:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T10:55:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:55:59,536 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125066492813b5e477fb4614920c95a7d07_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125066492813b5e477fb4614920c95a7d07_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:55:59,543 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/f0e4a49e1fa6449684d7fec51f3170d2, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:55:59,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/f0e4a49e1fa6449684d7fec51f3170d2 is 175, key is test_row_0/A:col10/1732532159003/Put/seqid=0 2024-11-25T10:55:59,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741972_1148 (size=39749) 2024-11-25T10:55:59,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:55:59,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:59,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532219744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532219745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532219745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532219746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:55:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532219746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:55:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:55:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:55:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T10:55:59,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:55:59,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:55:59,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:55:59,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:55:59,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:55:59,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/f0e4a49e1fa6449684d7fec51f3170d2 2024-11-25T10:56:00,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/62ef2c4fc00f487298df9f22b74d4e89 is 50, key is test_row_0/B:col10/1732532159003/Put/seqid=0 2024-11-25T10:56:00,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741973_1149 (size=12151) 2024-11-25T10:56:00,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/62ef2c4fc00f487298df9f22b74d4e89 2024-11-25T10:56:00,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:56:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:00,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/592ca637f73b41a9ab6974e4e2025293 is 50, key is test_row_0/C:col10/1732532159003/Put/seqid=0 2024-11-25T10:56:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:00,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741974_1150 (size=12151) 2024-11-25T10:56:00,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/592ca637f73b41a9ab6974e4e2025293 2024-11-25T10:56:00,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/f0e4a49e1fa6449684d7fec51f3170d2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/f0e4a49e1fa6449684d7fec51f3170d2 2024-11-25T10:56:00,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/f0e4a49e1fa6449684d7fec51f3170d2, entries=200, sequenceid=180, filesize=38.8 K 2024-11-25T10:56:00,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/62ef2c4fc00f487298df9f22b74d4e89 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/62ef2c4fc00f487298df9f22b74d4e89 2024-11-25T10:56:00,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/62ef2c4fc00f487298df9f22b74d4e89, entries=150, sequenceid=180, filesize=11.9 K 2024-11-25T10:56:00,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/592ca637f73b41a9ab6974e4e2025293 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/592ca637f73b41a9ab6974e4e2025293 2024-11-25T10:56:00,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532220254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T10:56:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532220266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:00,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532220266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:00,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532220269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532220272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:00,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:56:00,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:00,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:00,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:00,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/592ca637f73b41a9ab6974e4e2025293, entries=150, sequenceid=180, filesize=11.9 K 2024-11-25T10:56:00,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 78f10d0299f2958c87d34c58cfcb3253 in 1324ms, sequenceid=180, compaction requested=true 2024-11-25T10:56:00,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:00,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:00,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:00,330 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:00,331 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:00,334 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:00,334 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/A is initiating minor compaction (all files) 2024-11-25T10:56:00,334 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/A in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,334 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/43e6a61ef62844cba7f36bea5461fbbe, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/e5a572256c224022836ce3073c71b0a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/8a43fe3ed22a47c9a79273c0c55d4624, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/f0e4a49e1fa6449684d7fec51f3170d2] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=130.1 K 2024-11-25T10:56:00,334 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,334 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/43e6a61ef62844cba7f36bea5461fbbe, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/e5a572256c224022836ce3073c71b0a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/8a43fe3ed22a47c9a79273c0c55d4624, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/f0e4a49e1fa6449684d7fec51f3170d2] 2024-11-25T10:56:00,335 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e6a61ef62844cba7f36bea5461fbbe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732532154270 2024-11-25T10:56:00,336 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:00,336 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/B is initiating minor compaction (all files) 2024-11-25T10:56:00,336 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/B in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,336 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea4b6f8af55e4fe484257bcef2489b8e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/8ad99027aa644846b8a1eee42692f1a2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/1b091ef19fdd47e5bc5ae712c7465fcd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/62ef2c4fc00f487298df9f22b74d4e89] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=47.6 K 2024-11-25T10:56:00,337 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4b6f8af55e4fe484257bcef2489b8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732532154270 2024-11-25T10:56:00,337 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a572256c224022836ce3073c71b0a3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532155034 2024-11-25T10:56:00,337 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad99027aa644846b8a1eee42692f1a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532155034 2024-11-25T10:56:00,338 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a43fe3ed22a47c9a79273c0c55d4624, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732532155463 2024-11-25T10:56:00,338 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0e4a49e1fa6449684d7fec51f3170d2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732532157804 2024-11-25T10:56:00,345 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b091ef19fdd47e5bc5ae712c7465fcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732532155463 2024-11-25T10:56:00,345 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ef2c4fc00f487298df9f22b74d4e89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732532157804 2024-11-25T10:56:00,352 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:00,376 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125b9c834f2be7448e7a3dac35066a7150c_78f10d0299f2958c87d34c58cfcb3253 store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:00,380 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125b9c834f2be7448e7a3dac35066a7150c_78f10d0299f2958c87d34c58cfcb3253, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:00,380 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b9c834f2be7448e7a3dac35066a7150c_78f10d0299f2958c87d34c58cfcb3253 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:00,393 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#B#compaction#124 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:00,394 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/5697620dd2e04f14b62ab5e10f837813 is 50, key is test_row_0/B:col10/1732532159003/Put/seqid=0 2024-11-25T10:56:00,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T10:56:00,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,474 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T10:56:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:56:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:56:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:56:00,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741976_1152 (size=12561) 2024-11-25T10:56:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:00,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741975_1151 (size=4469) 2024-11-25T10:56:00,494 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#A#compaction#123 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:00,494 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/b79d438197074f76a3f47a7627ee7399 is 175, key is test_row_0/A:col10/1732532159003/Put/seqid=0 2024-11-25T10:56:00,517 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/5697620dd2e04f14b62ab5e10f837813 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5697620dd2e04f14b62ab5e10f837813 2024-11-25T10:56:00,529 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/B of 78f10d0299f2958c87d34c58cfcb3253 into 5697620dd2e04f14b62ab5e10f837813(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:00,529 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:00,529 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/B, priority=12, startTime=1732532160330; duration=0sec 2024-11-25T10:56:00,529 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:00,529 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:B 2024-11-25T10:56:00,529 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:00,531 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:00,531 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/C is initiating minor compaction (all files) 2024-11-25T10:56:00,531 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/C in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:00,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259bd85d6191924d388b4085e3a793cc9c_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532159089/Put/seqid=0 2024-11-25T10:56:00,531 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/aaa0420d1f2d49129ce49d5698ede242, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/04937fb736434b6eb22b7a2d503ce469, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/a78af616eaef4545b9e60d7b627ee571, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/592ca637f73b41a9ab6974e4e2025293] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=47.6 K 2024-11-25T10:56:00,532 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting aaa0420d1f2d49129ce49d5698ede242, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732532154270 2024-11-25T10:56:00,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 04937fb736434b6eb22b7a2d503ce469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532155034 2024-11-25T10:56:00,534 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a78af616eaef4545b9e60d7b627ee571, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732532155463 2024-11-25T10:56:00,537 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 592ca637f73b41a9ab6974e4e2025293, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732532157804 2024-11-25T10:56:00,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741977_1153 (size=31515) 2024-11-25T10:56:00,572 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/b79d438197074f76a3f47a7627ee7399 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b79d438197074f76a3f47a7627ee7399 2024-11-25T10:56:00,577 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:00,578 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/29bf23fbaaaa40fe93c7878c2cab11b4 is 50, key is test_row_0/C:col10/1732532159003/Put/seqid=0 2024-11-25T10:56:00,611 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/A of 78f10d0299f2958c87d34c58cfcb3253 into b79d438197074f76a3f47a7627ee7399(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:00,611 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:00,611 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/A, priority=12, startTime=1732532160330; duration=0sec 2024-11-25T10:56:00,611 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:00,611 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:A 2024-11-25T10:56:00,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741978_1154 (size=12304) 2024-11-25T10:56:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:00,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741979_1155 (size=12561) 2024-11-25T10:56:00,635 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259bd85d6191924d388b4085e3a793cc9c_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259bd85d6191924d388b4085e3a793cc9c_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a2ddf27360504dcfa76d51a099264d79, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:00,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a2ddf27360504dcfa76d51a099264d79 is 175, key is test_row_0/A:col10/1732532159089/Put/seqid=0 2024-11-25T10:56:00,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741980_1156 (size=31105) 2024-11-25T10:56:00,685 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a2ddf27360504dcfa76d51a099264d79 2024-11-25T10:56:00,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/592db97099744bfd9ee7bef7388523c0 is 50, key is test_row_0/B:col10/1732532159089/Put/seqid=0 2024-11-25T10:56:00,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741981_1157 (size=12151) 2024-11-25T10:56:00,798 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/592db97099744bfd9ee7bef7388523c0 2024-11-25T10:56:00,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0029d6f4f26145408cc6e7d2bc897d03 is 50, key is test_row_0/C:col10/1732532159089/Put/seqid=0 2024-11-25T10:56:00,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741982_1158 (size=12151) 2024-11-25T10:56:01,109 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/29bf23fbaaaa40fe93c7878c2cab11b4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/29bf23fbaaaa40fe93c7878c2cab11b4 2024-11-25T10:56:01,145 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/C of 78f10d0299f2958c87d34c58cfcb3253 into 29bf23fbaaaa40fe93c7878c2cab11b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:01,145 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:01,145 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/C, priority=12, startTime=1732532160331; duration=0sec 2024-11-25T10:56:01,150 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:01,150 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:C 2024-11-25T10:56:01,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:01,305 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0029d6f4f26145408cc6e7d2bc897d03 2024-11-25T10:56:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T10:56:01,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/a2ddf27360504dcfa76d51a099264d79 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a2ddf27360504dcfa76d51a099264d79 2024-11-25T10:56:01,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532221342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532221345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532221347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532221359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532221360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,389 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a2ddf27360504dcfa76d51a099264d79, entries=150, sequenceid=196, filesize=30.4 K 2024-11-25T10:56:01,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/592db97099744bfd9ee7bef7388523c0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/592db97099744bfd9ee7bef7388523c0 2024-11-25T10:56:01,439 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/592db97099744bfd9ee7bef7388523c0, entries=150, sequenceid=196, filesize=11.9 K 2024-11-25T10:56:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0029d6f4f26145408cc6e7d2bc897d03 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0029d6f4f26145408cc6e7d2bc897d03 2024-11-25T10:56:01,449 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0029d6f4f26145408cc6e7d2bc897d03, entries=150, sequenceid=196, filesize=11.9 K 2024-11-25T10:56:01,457 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 78f10d0299f2958c87d34c58cfcb3253 in 983ms, sequenceid=196, compaction requested=false 2024-11-25T10:56:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-25T10:56:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-25T10:56:01,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:56:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:56:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:56:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:56:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:01,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-25T10:56:01,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-25T10:56:01,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3410 sec 2024-11-25T10:56:01,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252be06c6940114690a8ae45de52205507_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532161470/Put/seqid=0 2024-11-25T10:56:01,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532221508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532221511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532221513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532221520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532221521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741983_1159 (size=17284) 2024-11-25T10:56:01,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:01,609 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252be06c6940114690a8ae45de52205507_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252be06c6940114690a8ae45de52205507_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:01,611 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/17221b215d8c4640a4bc9023a20a469a, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:01,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/17221b215d8c4640a4bc9023a20a469a is 175, key is test_row_0/A:col10/1732532161470/Put/seqid=0 2024-11-25T10:56:01,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532221641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532221641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741984_1160 (size=48389) 2024-11-25T10:56:01,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532221654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532221667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532221668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532221867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532221869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532221877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532221885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:01,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532221897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/17221b215d8c4640a4bc9023a20a469a 2024-11-25T10:56:02,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/79827b9e9ecc4f16a086a34f697f1ada is 50, key is test_row_0/B:col10/1732532161470/Put/seqid=0 2024-11-25T10:56:02,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741985_1161 (size=12151) 2024-11-25T10:56:02,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/79827b9e9ecc4f16a086a34f697f1ada 2024-11-25T10:56:02,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/ff7596668adb4b85a06f4675228e3350 is 50, key is test_row_0/C:col10/1732532161470/Put/seqid=0 2024-11-25T10:56:02,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532222184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532222193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532222194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532222197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532222214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741986_1162 (size=12151) 2024-11-25T10:56:02,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/ff7596668adb4b85a06f4675228e3350 2024-11-25T10:56:02,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/17221b215d8c4640a4bc9023a20a469a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/17221b215d8c4640a4bc9023a20a469a 2024-11-25T10:56:02,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/17221b215d8c4640a4bc9023a20a469a, entries=250, sequenceid=218, filesize=47.3 K 2024-11-25T10:56:02,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/79827b9e9ecc4f16a086a34f697f1ada as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/79827b9e9ecc4f16a086a34f697f1ada 2024-11-25T10:56:02,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532222709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532222719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532222720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532222720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/79827b9e9ecc4f16a086a34f697f1ada, entries=150, sequenceid=218, filesize=11.9 K 2024-11-25T10:56:02,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532222737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/ff7596668adb4b85a06f4675228e3350 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ff7596668adb4b85a06f4675228e3350 2024-11-25T10:56:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:02,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ff7596668adb4b85a06f4675228e3350, entries=150, sequenceid=218, filesize=11.9 K 2024-11-25T10:56:02,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 78f10d0299f2958c87d34c58cfcb3253 in 1342ms, sequenceid=218, compaction requested=true 2024-11-25T10:56:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:56:02,814 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:02,816 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:02,827 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:02,827 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/A is initiating minor compaction (all files) 2024-11-25T10:56:02,827 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/A in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:02,828 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b79d438197074f76a3f47a7627ee7399, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a2ddf27360504dcfa76d51a099264d79, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/17221b215d8c4640a4bc9023a20a469a] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=108.4 K 2024-11-25T10:56:02,828 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:02,828 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b79d438197074f76a3f47a7627ee7399, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a2ddf27360504dcfa76d51a099264d79, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/17221b215d8c4640a4bc9023a20a469a] 2024-11-25T10:56:02,828 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:02,828 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/C is initiating minor compaction (all files) 2024-11-25T10:56:02,828 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/C in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:02,828 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/29bf23fbaaaa40fe93c7878c2cab11b4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0029d6f4f26145408cc6e7d2bc897d03, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ff7596668adb4b85a06f4675228e3350] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=36.0 K 2024-11-25T10:56:02,829 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 29bf23fbaaaa40fe93c7878c2cab11b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732532157804 2024-11-25T10:56:02,829 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b79d438197074f76a3f47a7627ee7399, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732532157804 2024-11-25T10:56:02,830 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0029d6f4f26145408cc6e7d2bc897d03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732532159087 2024-11-25T10:56:02,830 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ddf27360504dcfa76d51a099264d79, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732532159087 2024-11-25T10:56:02,831 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7596668adb4b85a06f4675228e3350, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732532161344 2024-11-25T10:56:02,831 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17221b215d8c4640a4bc9023a20a469a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732532161340 2024-11-25T10:56:02,851 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:02,858 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#C#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:02,859 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/87a0da27fc8d4935af507ca15a062360 is 50, key is test_row_0/C:col10/1732532161470/Put/seqid=0 2024-11-25T10:56:02,863 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125feb72dae0c5141e0bc61f619774061be_78f10d0299f2958c87d34c58cfcb3253 store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:02,866 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125feb72dae0c5141e0bc61f619774061be_78f10d0299f2958c87d34c58cfcb3253, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:02,866 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125feb72dae0c5141e0bc61f619774061be_78f10d0299f2958c87d34c58cfcb3253 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:02,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741987_1163 (size=12663) 2024-11-25T10:56:02,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741988_1164 (size=4469) 2024-11-25T10:56:02,992 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#A#compaction#132 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:02,993 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/4ee734d5c5094a97858724e32487ef8c is 175, key is test_row_0/A:col10/1732532161470/Put/seqid=0 2024-11-25T10:56:03,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741989_1165 (size=31617) 2024-11-25T10:56:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T10:56:03,314 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:03,315 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-25T10:56:03,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:03,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:03,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T10:56:03,366 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/87a0da27fc8d4935af507ca15a062360 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/87a0da27fc8d4935af507ca15a062360 2024-11-25T10:56:03,389 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/C of 78f10d0299f2958c87d34c58cfcb3253 into 87a0da27fc8d4935af507ca15a062360(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:03,389 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:03,389 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/C, priority=13, startTime=1732532162814; duration=0sec 2024-11-25T10:56:03,390 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:03,390 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:C 2024-11-25T10:56:03,390 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:03,404 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:03,404 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/B is initiating minor compaction (all files) 2024-11-25T10:56:03,405 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/B in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:03,405 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5697620dd2e04f14b62ab5e10f837813, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/592db97099744bfd9ee7bef7388523c0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/79827b9e9ecc4f16a086a34f697f1ada] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=36.0 K 2024-11-25T10:56:03,406 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5697620dd2e04f14b62ab5e10f837813, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732532157804 2024-11-25T10:56:03,411 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 592db97099744bfd9ee7bef7388523c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732532159087 2024-11-25T10:56:03,412 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 79827b9e9ecc4f16a086a34f697f1ada, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732532161344 2024-11-25T10:56:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T10:56:03,463 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#B#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:03,463 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/018aaf1c23274c98b865b7bb5c2cc0a3 is 50, key is test_row_0/B:col10/1732532161470/Put/seqid=0 2024-11-25T10:56:03,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T10:56:03,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:03,473 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:56:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:56:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:56:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:56:03,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:03,480 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/4ee734d5c5094a97858724e32487ef8c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/4ee734d5c5094a97858724e32487ef8c 2024-11-25T10:56:03,501 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/A of 78f10d0299f2958c87d34c58cfcb3253 into 4ee734d5c5094a97858724e32487ef8c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:03,501 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:03,501 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/A, priority=13, startTime=1732532162813; duration=0sec 2024-11-25T10:56:03,501 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:03,501 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:A 2024-11-25T10:56:03,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741990_1166 (size=12663) 2024-11-25T10:56:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125818bb5be563546f48763688f3b1bd116_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532161511/Put/seqid=0 2024-11-25T10:56:03,557 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/018aaf1c23274c98b865b7bb5c2cc0a3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/018aaf1c23274c98b865b7bb5c2cc0a3 2024-11-25T10:56:03,579 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/B of 78f10d0299f2958c87d34c58cfcb3253 into 018aaf1c23274c98b865b7bb5c2cc0a3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:03,579 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:03,579 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/B, priority=13, startTime=1732532162814; duration=0sec 2024-11-25T10:56:03,579 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:03,579 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:B 2024-11-25T10:56:03,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741991_1167 (size=12304) 2024-11-25T10:56:03,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:03,624 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125818bb5be563546f48763688f3b1bd116_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125818bb5be563546f48763688f3b1bd116_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:03,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/01dee952ac0a43a1aca2b959116b1ed6, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T10:56:03,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/01dee952ac0a43a1aca2b959116b1ed6 is 175, key is test_row_0/A:col10/1732532161511/Put/seqid=0 2024-11-25T10:56:03,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741992_1168 (size=31105) 2024-11-25T10:56:03,712 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/01dee952ac0a43a1aca2b959116b1ed6 2024-11-25T10:56:03,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/a9e80bfdc79044abad9f542fb6b43132 is 50, key is test_row_0/B:col10/1732532161511/Put/seqid=0 2024-11-25T10:56:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:03,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:03,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741993_1169 (size=12151) 2024-11-25T10:56:03,808 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/a9e80bfdc79044abad9f542fb6b43132 2024-11-25T10:56:03,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532223816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532223817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:03,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532223837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:03,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0092184dbd4740d78d51593a97c588bd is 50, key is test_row_0/C:col10/1732532161511/Put/seqid=0 2024-11-25T10:56:03,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532223840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532223860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:03,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741994_1170 (size=12151) 2024-11-25T10:56:03,949 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0092184dbd4740d78d51593a97c588bd 2024-11-25T10:56:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T10:56:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532223957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:03,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532223977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/01dee952ac0a43a1aca2b959116b1ed6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/01dee952ac0a43a1aca2b959116b1ed6 2024-11-25T10:56:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532223977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532223985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532223994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,030 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/01dee952ac0a43a1aca2b959116b1ed6, entries=150, sequenceid=232, filesize=30.4 K 2024-11-25T10:56:04,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/a9e80bfdc79044abad9f542fb6b43132 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a9e80bfdc79044abad9f542fb6b43132 2024-11-25T10:56:04,056 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a9e80bfdc79044abad9f542fb6b43132, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T10:56:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/0092184dbd4740d78d51593a97c588bd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0092184dbd4740d78d51593a97c588bd 2024-11-25T10:56:04,086 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0092184dbd4740d78d51593a97c588bd, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T10:56:04,088 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 78f10d0299f2958c87d34c58cfcb3253 in 615ms, sequenceid=232, compaction requested=false 2024-11-25T10:56:04,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:04,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:04,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-25T10:56:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-25T10:56:04,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-25T10:56:04,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 776 msec 2024-11-25T10:56:04,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 784 msec 2024-11-25T10:56:04,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532163813, max timestamp=1732532164188Num uniques 859; 2024-11-25T10:56:04,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:04,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532163813, max timestamp=1732532164188Num uniques 859; 2024-11-25T10:56:04,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:04,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-25T10:56:04,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:56:04,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:04,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:56:04,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:04,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:56:04,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:04,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ef324b8746f24ae49f62f982a83e15f9_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532163835/Put/seqid=0 2024-11-25T10:56:04,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741995_1171 (size=12304) 2024-11-25T10:56:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T10:56:04,457 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:04,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-25T10:56:04,465 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:04,466 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:04,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T10:56:04,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532164319, max timestamp=1732532164486Num uniques 859; 2024-11-25T10:56:04,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:04,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532164319, max timestamp=1732532164486Num uniques 859; 2024-11-25T10:56:04,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:04,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532164319, max timestamp=1732532164486Num uniques 859; 2024-11-25T10:56:04,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T10:56:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532224567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532224569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532224570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532224571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532224574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:04,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:04,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:04,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:04,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532224697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532224704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532224704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532224704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532224724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:04,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:04,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T10:56:04,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ef324b8746f24ae49f62f982a83e15f9_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ef324b8746f24ae49f62f982a83e15f9_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:04,842 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/aa861b66fd9f4299b0e1de7e8e7bb99b, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:04,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/aa861b66fd9f4299b0e1de7e8e7bb99b is 175, key is test_row_0/A:col10/1732532163835/Put/seqid=0 2024-11-25T10:56:04,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741996_1172 (size=31105) 2024-11-25T10:56:04,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532224917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:04,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:04,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:04,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:04,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:04,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532224925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532224925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532224925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:04,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:04,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532224983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:05,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:05,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T10:56:05,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532225245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,254 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/aa861b66fd9f4299b0e1de7e8e7bb99b 2024-11-25T10:56:05,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:05,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532225255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532225255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532225256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/da2349044dcc4da78985c6803281f221 is 50, key is test_row_0/B:col10/1732532163835/Put/seqid=0 2024-11-25T10:56:05,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532225305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741997_1173 (size=12151) 2024-11-25T10:56:05,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T10:56:05,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/da2349044dcc4da78985c6803281f221 2024-11-25T10:56:05,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532225769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532225777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:05,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532225788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/86648ad71cce463fa443bcba1ea4f97f is 50, key is test_row_0/C:col10/1732532163835/Put/seqid=0 2024-11-25T10:56:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532225789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532225821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:05,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741998_1174 (size=12151) 2024-11-25T10:56:05,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:05,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:05,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:05,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:06,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:06,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/86648ad71cce463fa443bcba1ea4f97f 2024-11-25T10:56:06,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:06,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:06,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:06,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:06,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/aa861b66fd9f4299b0e1de7e8e7bb99b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/aa861b66fd9f4299b0e1de7e8e7bb99b 2024-11-25T10:56:06,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/aa861b66fd9f4299b0e1de7e8e7bb99b, entries=150, sequenceid=255, filesize=30.4 K 2024-11-25T10:56:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/da2349044dcc4da78985c6803281f221 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/da2349044dcc4da78985c6803281f221 2024-11-25T10:56:06,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/da2349044dcc4da78985c6803281f221, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T10:56:06,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/86648ad71cce463fa443bcba1ea4f97f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/86648ad71cce463fa443bcba1ea4f97f 2024-11-25T10:56:06,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/86648ad71cce463fa443bcba1ea4f97f, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T10:56:06,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 78f10d0299f2958c87d34c58cfcb3253 in 2137ms, sequenceid=255, compaction requested=true 2024-11-25T10:56:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:06,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:06,354 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:06,354 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:06,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:06,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:06,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:06,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:06,361 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:06,361 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/B is initiating minor compaction (all files) 2024-11-25T10:56:06,361 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/B in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,361 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/018aaf1c23274c98b865b7bb5c2cc0a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a9e80bfdc79044abad9f542fb6b43132, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/da2349044dcc4da78985c6803281f221] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=36.1 K 2024-11-25T10:56:06,361 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:06,361 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/A is initiating minor compaction (all files) 2024-11-25T10:56:06,361 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/A in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,362 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/4ee734d5c5094a97858724e32487ef8c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/01dee952ac0a43a1aca2b959116b1ed6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/aa861b66fd9f4299b0e1de7e8e7bb99b] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=91.6 K 2024-11-25T10:56:06,362 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,362 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/4ee734d5c5094a97858724e32487ef8c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/01dee952ac0a43a1aca2b959116b1ed6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/aa861b66fd9f4299b0e1de7e8e7bb99b] 2024-11-25T10:56:06,362 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 018aaf1c23274c98b865b7bb5c2cc0a3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732532161344 2024-11-25T10:56:06,363 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee734d5c5094a97858724e32487ef8c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732532161344 2024-11-25T10:56:06,363 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e80bfdc79044abad9f542fb6b43132, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732532161511 2024-11-25T10:56:06,363 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01dee952ac0a43a1aca2b959116b1ed6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732532161511 2024-11-25T10:56:06,364 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting da2349044dcc4da78985c6803281f221, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732532163835 2024-11-25T10:56:06,364 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa861b66fd9f4299b0e1de7e8e7bb99b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732532163835 2024-11-25T10:56:06,375 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:06,391 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:06,392 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2efaebf6743c45c9977a674fa89fcf1a is 50, key is test_row_0/B:col10/1732532163835/Put/seqid=0 2024-11-25T10:56:06,398 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ca0f6195f14c438c8ff6b7dd1b78b766_78f10d0299f2958c87d34c58cfcb3253 store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:06,400 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ca0f6195f14c438c8ff6b7dd1b78b766_78f10d0299f2958c87d34c58cfcb3253, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:06,401 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ca0f6195f14c438c8ff6b7dd1b78b766_78f10d0299f2958c87d34c58cfcb3253 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:06,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741999_1175 (size=12765) 2024-11-25T10:56:06,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T10:56:06,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,457 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-25T10:56:06,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:56:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:56:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:56:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:06,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742000_1176 (size=4469) 2024-11-25T10:56:06,469 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#A#compaction#141 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:06,470 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/27d8be68dd2746bab30ba934b35cb1a9 is 175, key is test_row_0/A:col10/1732532163835/Put/seqid=0 2024-11-25T10:56:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112574cafe5c3d354d13a1eb98827dcc2165_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532164558/Put/seqid=0 2024-11-25T10:56:06,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742001_1177 (size=31719) 2024-11-25T10:56:06,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742002_1178 (size=12454) 2024-11-25T10:56:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:06,562 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112574cafe5c3d354d13a1eb98827dcc2165_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112574cafe5c3d354d13a1eb98827dcc2165_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:06,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/145b5ea277a5485fb73d79275f051f5c, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:06,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/145b5ea277a5485fb73d79275f051f5c is 175, key is test_row_0/A:col10/1732532164558/Put/seqid=0 2024-11-25T10:56:06,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742003_1179 (size=31255) 2024-11-25T10:56:06,633 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/145b5ea277a5485fb73d79275f051f5c 2024-11-25T10:56:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T10:56:06,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/17cc458cafc14801ab5a24753c10884b is 50, key is test_row_0/B:col10/1732532164558/Put/seqid=0 2024-11-25T10:56:06,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742004_1180 (size=12301) 2024-11-25T10:56:06,753 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/17cc458cafc14801ab5a24753c10884b 2024-11-25T10:56:06,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. as already flushing 2024-11-25T10:56:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:06,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/9e5f9e83add141e3bcf4de579205a10c is 50, key is test_row_0/C:col10/1732532164558/Put/seqid=0 2024-11-25T10:56:06,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742005_1181 (size=12301) 2024-11-25T10:56:06,848 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/2efaebf6743c45c9977a674fa89fcf1a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2efaebf6743c45c9977a674fa89fcf1a 2024-11-25T10:56:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532226837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:06,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532226837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:06,856 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/B of 78f10d0299f2958c87d34c58cfcb3253 into 2efaebf6743c45c9977a674fa89fcf1a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:06,856 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:06,856 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/B, priority=13, startTime=1732532166354; duration=0sec 2024-11-25T10:56:06,856 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:06,856 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:B 2024-11-25T10:56:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,857 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:06,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49718 deadline: 1732532226845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:06,865 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in 78f10d0299f2958c87d34c58cfcb3253 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2293 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:06,873 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:06,873 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/C is initiating minor compaction (all files) 2024-11-25T10:56:06,873 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/C in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:06,873 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/87a0da27fc8d4935af507ca15a062360, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0092184dbd4740d78d51593a97c588bd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/86648ad71cce463fa443bcba1ea4f97f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=36.1 K 2024-11-25T10:56:06,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532226855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:06,877 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a0da27fc8d4935af507ca15a062360, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732532161344 2024-11-25T10:56:06,882 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0092184dbd4740d78d51593a97c588bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732532161511 2024-11-25T10:56:06,887 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 86648ad71cce463fa443bcba1ea4f97f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732532163835 2024-11-25T10:56:06,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532226865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:06,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:06,906 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:06,907 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/388275fc560645258b7f8192d31ac1f6 is 50, key is test_row_0/C:col10/1732532163835/Put/seqid=0 2024-11-25T10:56:06,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742006_1182 (size=12765) 2024-11-25T10:56:07,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532227000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532227001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532227001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532227005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,020 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/27d8be68dd2746bab30ba934b35cb1a9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/27d8be68dd2746bab30ba934b35cb1a9 2024-11-25T10:56:07,037 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/A of 78f10d0299f2958c87d34c58cfcb3253 into 27d8be68dd2746bab30ba934b35cb1a9(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:07,037 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:07,037 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/A, priority=13, startTime=1732532166354; duration=0sec 2024-11-25T10:56:07,037 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:07,037 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:A 2024-11-25T10:56:07,038 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/388275fc560645258b7f8192d31ac1f6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/388275fc560645258b7f8192d31ac1f6 2024-11-25T10:56:07,049 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/C of 78f10d0299f2958c87d34c58cfcb3253 into 388275fc560645258b7f8192d31ac1f6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:07,049 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:07,049 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/C, priority=13, startTime=1732532166355; duration=0sec 2024-11-25T10:56:07,049 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:07,049 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:C 2024-11-25T10:56:07,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49690 deadline: 1732532227205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49662 deadline: 1732532227220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49640 deadline: 1732532227224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,230 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/9e5f9e83add141e3bcf4de579205a10c 2024-11-25T10:56:07,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/145b5ea277a5485fb73d79275f051f5c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/145b5ea277a5485fb73d79275f051f5c 2024-11-25T10:56:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49648 deadline: 1732532227225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:07,242 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/145b5ea277a5485fb73d79275f051f5c, entries=150, sequenceid=290, filesize=30.5 K 2024-11-25T10:56:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/17cc458cafc14801ab5a24753c10884b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/17cc458cafc14801ab5a24753c10884b 2024-11-25T10:56:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:07,269 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/17cc458cafc14801ab5a24753c10884b, entries=150, sequenceid=290, filesize=12.0 K 2024-11-25T10:56:07,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/9e5f9e83add141e3bcf4de579205a10c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/9e5f9e83add141e3bcf4de579205a10c 2024-11-25T10:56:07,326 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/9e5f9e83add141e3bcf4de579205a10c, entries=150, sequenceid=290, filesize=12.0 K 2024-11-25T10:56:07,332 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 78f10d0299f2958c87d34c58cfcb3253 in 875ms, sequenceid=290, compaction requested=false 2024-11-25T10:56:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-25T10:56:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-25T10:56:07,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-25T10:56:07,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8690 sec 2024-11-25T10:56:07,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8950 sec 2024-11-25T10:56:07,418 INFO [Thread-551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-551. 2024-11-25T10:56:07,418 DEBUG [Thread-551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,418 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,418 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,421 INFO [Thread-553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-553. 2024-11-25T10:56:07,421 DEBUG [Thread-553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,421 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,421 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,426 INFO [Thread-557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-557. 2024-11-25T10:56:07,426 DEBUG [Thread-557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,426 DEBUG [Thread-557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,426 DEBUG [Thread-557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,443 INFO [Thread-555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-555. 2024-11-25T10:56:07,443 DEBUG [Thread-555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,443 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,443 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,465 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T10:56:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:07,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T10:56:07,527 INFO [Thread-548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-548. 2024-11-25T10:56:07,527 DEBUG [Thread-548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,528 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,528 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:56:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:56:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:56:07,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:07,541 INFO [Thread-546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-546. 2024-11-25T10:56:07,541 DEBUG [Thread-546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,541 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,541 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,545 INFO [Thread-542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-542. 2024-11-25T10:56:07,545 DEBUG [Thread-542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,545 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,545 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,555 INFO [Thread-544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-544. 2024-11-25T10:56:07,556 DEBUG [Thread-544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:07,556 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,556 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:07,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:07,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112579a33a8c91324b87b2fa51604a92327a_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532167525/Put/seqid=0 2024-11-25T10:56:07,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742007_1183 (size=12454) 2024-11-25T10:56:07,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:07,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112579a33a8c91324b87b2fa51604a92327a_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112579a33a8c91324b87b2fa51604a92327a_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:07,609 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/004828260cbf4553892262808d444094, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:07,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/004828260cbf4553892262808d444094 is 175, key is test_row_0/A:col10/1732532167525/Put/seqid=0 2024-11-25T10:56:07,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742008_1184 (size=31255) 2024-11-25T10:56:07,621 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/004828260cbf4553892262808d444094 2024-11-25T10:56:07,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/a4a5321f532d4822b6e16c38601e811c is 50, key is test_row_0/B:col10/1732532167525/Put/seqid=0 2024-11-25T10:56:07,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742009_1185 (size=12301) 2024-11-25T10:56:08,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/a4a5321f532d4822b6e16c38601e811c 2024-11-25T10:56:08,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/31f190a5111143b19cd0df125ea7b789 is 50, key is test_row_0/C:col10/1732532167525/Put/seqid=0 2024-11-25T10:56:08,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742010_1186 (size=12301) 2024-11-25T10:56:08,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/31f190a5111143b19cd0df125ea7b789 2024-11-25T10:56:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/004828260cbf4553892262808d444094 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/004828260cbf4553892262808d444094 2024-11-25T10:56:08,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/004828260cbf4553892262808d444094, entries=150, sequenceid=305, filesize=30.5 K 2024-11-25T10:56:08,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/a4a5321f532d4822b6e16c38601e811c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a4a5321f532d4822b6e16c38601e811c 2024-11-25T10:56:08,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a4a5321f532d4822b6e16c38601e811c, entries=150, sequenceid=305, filesize=12.0 K 2024-11-25T10:56:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/31f190a5111143b19cd0df125ea7b789 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/31f190a5111143b19cd0df125ea7b789 2024-11-25T10:56:08,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/31f190a5111143b19cd0df125ea7b789, entries=150, sequenceid=305, filesize=12.0 K 2024-11-25T10:56:08,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 78f10d0299f2958c87d34c58cfcb3253 in 1017ms, sequenceid=305, compaction requested=true 2024-11-25T10:56:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:08,545 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78f10d0299f2958c87d34c58cfcb3253:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:08,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:08,545 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:08,546 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:08,546 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/A is initiating minor compaction (all files) 2024-11-25T10:56:08,546 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/A in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:08,546 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/27d8be68dd2746bab30ba934b35cb1a9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/145b5ea277a5485fb73d79275f051f5c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/004828260cbf4553892262808d444094] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=92.0 K 2024-11-25T10:56:08,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:08,547 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:08,547 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/27d8be68dd2746bab30ba934b35cb1a9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/145b5ea277a5485fb73d79275f051f5c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/004828260cbf4553892262808d444094] 2024-11-25T10:56:08,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/B is initiating minor compaction (all files) 2024-11-25T10:56:08,547 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/B in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:08,547 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2efaebf6743c45c9977a674fa89fcf1a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/17cc458cafc14801ab5a24753c10884b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a4a5321f532d4822b6e16c38601e811c] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=36.5 K 2024-11-25T10:56:08,547 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d8be68dd2746bab30ba934b35cb1a9, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732532163835 2024-11-25T10:56:08,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efaebf6743c45c9977a674fa89fcf1a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732532163835 2024-11-25T10:56:08,548 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145b5ea277a5485fb73d79275f051f5c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732532164558 2024-11-25T10:56:08,548 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 17cc458cafc14801ab5a24753c10884b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732532164558 2024-11-25T10:56:08,548 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004828260cbf4553892262808d444094, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732532166832 2024-11-25T10:56:08,548 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a5321f532d4822b6e16c38601e811c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732532166832 2024-11-25T10:56:08,563 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:08,563 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/cd6003cdd2f9436b851264f13cdbf773 is 50, key is test_row_0/B:col10/1732532167525/Put/seqid=0 2024-11-25T10:56:08,582 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:08,607 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112566e84410b2d046ba84903987c8c9d08e_78f10d0299f2958c87d34c58cfcb3253 store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:08,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742011_1187 (size=13017) 2024-11-25T10:56:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T10:56:08,658 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:08,696 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112566e84410b2d046ba84903987c8c9d08e_78f10d0299f2958c87d34c58cfcb3253, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:08,696 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112566e84410b2d046ba84903987c8c9d08e_78f10d0299f2958c87d34c58cfcb3253 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:08,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742012_1188 (size=4469) 2024-11-25T10:56:08,878 INFO [Thread-540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-540. 2024-11-25T10:56:08,878 DEBUG [Thread-540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:08,878 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:08,878 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:08,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1332 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1277 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 617 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1851 rows 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 607 2024-11-25T10:56:08,879 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1821 rows 2024-11-25T10:56:08,879 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:56:08,879 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:56:08,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:08,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:08,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:08,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T10:56:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T10:56:08,899 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532168898"}]},"ts":"1732532168898"} 2024-11-25T10:56:08,914 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T10:56:08,915 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T10:56:08,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:56:08,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, UNASSIGN}] 2024-11-25T10:56:08,929 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, UNASSIGN 2024-11-25T10:56:08,937 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:08,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, UNASSIGN because future has completed 2024-11-25T10:56:08,941 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:56:08,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:56:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T10:56:09,040 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/cd6003cdd2f9436b851264f13cdbf773 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cd6003cdd2f9436b851264f13cdbf773 2024-11-25T10:56:09,069 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/B of 78f10d0299f2958c87d34c58cfcb3253 into cd6003cdd2f9436b851264f13cdbf773(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:09,069 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:09,069 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/B, priority=13, startTime=1732532168545; duration=0sec 2024-11-25T10:56:09,069 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:09,069 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:B 2024-11-25T10:56:09,069 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:09,076 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:09,077 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 78f10d0299f2958c87d34c58cfcb3253/C is initiating minor compaction (all files) 2024-11-25T10:56:09,078 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78f10d0299f2958c87d34c58cfcb3253/C in TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:09,078 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/388275fc560645258b7f8192d31ac1f6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/9e5f9e83add141e3bcf4de579205a10c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/31f190a5111143b19cd0df125ea7b789] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp, totalSize=36.5 K 2024-11-25T10:56:09,093 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 388275fc560645258b7f8192d31ac1f6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732532163835 2024-11-25T10:56:09,102 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e5f9e83add141e3bcf4de579205a10c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732532164558 2024-11-25T10:56:09,105 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f190a5111143b19cd0df125ea7b789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732532166832 2024-11-25T10:56:09,108 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:09,108 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:56:09,109 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 78f10d0299f2958c87d34c58cfcb3253, disabling compactions & flushes 2024-11-25T10:56:09,109 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:09,149 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:09,150 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/d23bdff3a5364fa7b50283ba73d44c7b is 50, key is test_row_0/C:col10/1732532167525/Put/seqid=0 2024-11-25T10:56:09,151 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78f10d0299f2958c87d34c58cfcb3253#A#compaction#151 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:09,151 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/0f3246730a944b96a5f383ff8bc9e3a5 is 175, key is test_row_0/A:col10/1732532167525/Put/seqid=0 2024-11-25T10:56:09,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742014_1190 (size=13017) 2024-11-25T10:56:09,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742013_1189 (size=31971) 2024-11-25T10:56:09,179 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/0f3246730a944b96a5f383ff8bc9e3a5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/0f3246730a944b96a5f383ff8bc9e3a5 2024-11-25T10:56:09,180 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/d23bdff3a5364fa7b50283ba73d44c7b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/d23bdff3a5364fa7b50283ba73d44c7b 2024-11-25T10:56:09,201 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/C of 78f10d0299f2958c87d34c58cfcb3253 into d23bdff3a5364fa7b50283ba73d44c7b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:09,201 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:09,201 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/C, priority=13, startTime=1732532168545; duration=0sec 2024-11-25T10:56:09,202 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:09,202 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:C 2024-11-25T10:56:09,205 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78f10d0299f2958c87d34c58cfcb3253/A of 78f10d0299f2958c87d34c58cfcb3253 into 0f3246730a944b96a5f383ff8bc9e3a5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:09,205 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78f10d0299f2958c87d34c58cfcb3253: 2024-11-25T10:56:09,205 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., storeName=78f10d0299f2958c87d34c58cfcb3253/A, priority=13, startTime=1732532168545; duration=0sec 2024-11-25T10:56:09,205 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:09,205 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:09,205 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78f10d0299f2958c87d34c58cfcb3253:A 2024-11-25T10:56:09,205 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:09,205 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. after waiting 0 ms 2024-11-25T10:56:09,205 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:09,206 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 78f10d0299f2958c87d34c58cfcb3253 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-25T10:56:09,206 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=A 2024-11-25T10:56:09,206 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:09,206 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=B 2024-11-25T10:56:09,206 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:09,206 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78f10d0299f2958c87d34c58cfcb3253, store=C 2024-11-25T10:56:09,206 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T10:56:09,221 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e9cdd12f9cd64c95b30d4882a84ab68e_78f10d0299f2958c87d34c58cfcb3253 is 50, key is test_row_0/A:col10/1732532168873/Put/seqid=0 2024-11-25T10:56:09,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742015_1191 (size=9914) 2024-11-25T10:56:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T10:56:09,635 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:09,658 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e9cdd12f9cd64c95b30d4882a84ab68e_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e9cdd12f9cd64c95b30d4882a84ab68e_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:09,659 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/9d47f6d068154587879fb5a438cfae57, store: [table=TestAcidGuarantees family=A region=78f10d0299f2958c87d34c58cfcb3253] 2024-11-25T10:56:09,660 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/9d47f6d068154587879fb5a438cfae57 is 175, key is test_row_0/A:col10/1732532168873/Put/seqid=0 2024-11-25T10:56:09,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742016_1192 (size=22561) 2024-11-25T10:56:09,701 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/9d47f6d068154587879fb5a438cfae57 2024-11-25T10:56:09,747 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/6f90ad05d8e24d328f0f34bfd0f321ae is 50, key is test_row_0/B:col10/1732532168873/Put/seqid=0 2024-11-25T10:56:09,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742017_1193 (size=9857) 2024-11-25T10:56:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T10:56:10,189 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/6f90ad05d8e24d328f0f34bfd0f321ae 2024-11-25T10:56:10,201 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/4e8c90874fdf4996bf9855c94266b1d6 is 50, key is test_row_0/C:col10/1732532168873/Put/seqid=0 2024-11-25T10:56:10,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742018_1194 (size=9857) 2024-11-25T10:56:10,230 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/4e8c90874fdf4996bf9855c94266b1d6 2024-11-25T10:56:10,242 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/A/9d47f6d068154587879fb5a438cfae57 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/9d47f6d068154587879fb5a438cfae57 2024-11-25T10:56:10,248 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/9d47f6d068154587879fb5a438cfae57, entries=100, sequenceid=315, filesize=22.0 K 2024-11-25T10:56:10,249 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/B/6f90ad05d8e24d328f0f34bfd0f321ae as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/6f90ad05d8e24d328f0f34bfd0f321ae 2024-11-25T10:56:10,255 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/6f90ad05d8e24d328f0f34bfd0f321ae, entries=100, sequenceid=315, filesize=9.6 K 2024-11-25T10:56:10,256 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/.tmp/C/4e8c90874fdf4996bf9855c94266b1d6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4e8c90874fdf4996bf9855c94266b1d6 2024-11-25T10:56:10,262 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4e8c90874fdf4996bf9855c94266b1d6, entries=100, sequenceid=315, filesize=9.6 K 2024-11-25T10:56:10,263 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 78f10d0299f2958c87d34c58cfcb3253 in 1058ms, sequenceid=315, compaction requested=false 2024-11-25T10:56:10,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/02548c4803f64acfadbeba509d7fc131, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/7a759055b742413587da2483cc1089bf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a17f730745074ffea936781ccb248470, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b1aed88ff229401eab698225164fb664, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/1975b70068e24dea8b9d02e0807d89cd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a43d6261841e4c2cbb162f1dfe2ab577, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/c65967c541234556be995b7e9e2b8b1f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/43e6a61ef62844cba7f36bea5461fbbe, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/014e8dfc29a04efe8da98dfcd6585d81, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/e5a572256c224022836ce3073c71b0a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/8a43fe3ed22a47c9a79273c0c55d4624, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/f0e4a49e1fa6449684d7fec51f3170d2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b79d438197074f76a3f47a7627ee7399, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a2ddf27360504dcfa76d51a099264d79, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/17221b215d8c4640a4bc9023a20a469a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/4ee734d5c5094a97858724e32487ef8c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/01dee952ac0a43a1aca2b959116b1ed6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/27d8be68dd2746bab30ba934b35cb1a9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/aa861b66fd9f4299b0e1de7e8e7bb99b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/145b5ea277a5485fb73d79275f051f5c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/004828260cbf4553892262808d444094] to archive 2024-11-25T10:56:10,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:56:10,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/02548c4803f64acfadbeba509d7fc131 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/02548c4803f64acfadbeba509d7fc131 2024-11-25T10:56:10,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/7a759055b742413587da2483cc1089bf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/7a759055b742413587da2483cc1089bf 2024-11-25T10:56:10,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a17f730745074ffea936781ccb248470 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a17f730745074ffea936781ccb248470 2024-11-25T10:56:10,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b1aed88ff229401eab698225164fb664 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b1aed88ff229401eab698225164fb664 2024-11-25T10:56:10,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/1975b70068e24dea8b9d02e0807d89cd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/1975b70068e24dea8b9d02e0807d89cd 2024-11-25T10:56:10,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a43d6261841e4c2cbb162f1dfe2ab577 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a43d6261841e4c2cbb162f1dfe2ab577 2024-11-25T10:56:10,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/c65967c541234556be995b7e9e2b8b1f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/c65967c541234556be995b7e9e2b8b1f 2024-11-25T10:56:10,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/43e6a61ef62844cba7f36bea5461fbbe to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/43e6a61ef62844cba7f36bea5461fbbe 2024-11-25T10:56:10,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/014e8dfc29a04efe8da98dfcd6585d81 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/014e8dfc29a04efe8da98dfcd6585d81 2024-11-25T10:56:10,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/e5a572256c224022836ce3073c71b0a3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/e5a572256c224022836ce3073c71b0a3 2024-11-25T10:56:10,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/8a43fe3ed22a47c9a79273c0c55d4624 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/8a43fe3ed22a47c9a79273c0c55d4624 2024-11-25T10:56:10,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/f0e4a49e1fa6449684d7fec51f3170d2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/f0e4a49e1fa6449684d7fec51f3170d2 2024-11-25T10:56:10,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b79d438197074f76a3f47a7627ee7399 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/b79d438197074f76a3f47a7627ee7399 2024-11-25T10:56:10,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a2ddf27360504dcfa76d51a099264d79 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/a2ddf27360504dcfa76d51a099264d79 2024-11-25T10:56:10,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/17221b215d8c4640a4bc9023a20a469a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/17221b215d8c4640a4bc9023a20a469a 2024-11-25T10:56:10,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/4ee734d5c5094a97858724e32487ef8c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/4ee734d5c5094a97858724e32487ef8c 2024-11-25T10:56:10,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/01dee952ac0a43a1aca2b959116b1ed6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/01dee952ac0a43a1aca2b959116b1ed6 2024-11-25T10:56:10,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/27d8be68dd2746bab30ba934b35cb1a9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/27d8be68dd2746bab30ba934b35cb1a9 2024-11-25T10:56:10,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/aa861b66fd9f4299b0e1de7e8e7bb99b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/aa861b66fd9f4299b0e1de7e8e7bb99b 2024-11-25T10:56:10,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/145b5ea277a5485fb73d79275f051f5c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/145b5ea277a5485fb73d79275f051f5c 2024-11-25T10:56:10,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/004828260cbf4553892262808d444094 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/004828260cbf4553892262808d444094 2024-11-25T10:56:10,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea69b18f4c0c4508b3af4e5b301b6fc4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/d7f4ee56e7f641afa5a5fae66837a932, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cb6cd7f4b83e41159e5bd6e55278bffd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/e400c9c2aad947688194b40ae7356eff, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2e657c464d0349359380d419d7f1a47e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/b3c0865bab4644b0baafc62fa809cbf8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5f8a3ecfbda042c4a87d9ba0182f2c2b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea4b6f8af55e4fe484257bcef2489b8e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2066118267354f87806f56786b53092e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/8ad99027aa644846b8a1eee42692f1a2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/1b091ef19fdd47e5bc5ae712c7465fcd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5697620dd2e04f14b62ab5e10f837813, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/62ef2c4fc00f487298df9f22b74d4e89, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/592db97099744bfd9ee7bef7388523c0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/018aaf1c23274c98b865b7bb5c2cc0a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/79827b9e9ecc4f16a086a34f697f1ada, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a9e80bfdc79044abad9f542fb6b43132, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2efaebf6743c45c9977a674fa89fcf1a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/da2349044dcc4da78985c6803281f221, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/17cc458cafc14801ab5a24753c10884b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a4a5321f532d4822b6e16c38601e811c] to archive 2024-11-25T10:56:10,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:56:10,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea69b18f4c0c4508b3af4e5b301b6fc4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea69b18f4c0c4508b3af4e5b301b6fc4 2024-11-25T10:56:10,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/d7f4ee56e7f641afa5a5fae66837a932 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/d7f4ee56e7f641afa5a5fae66837a932 2024-11-25T10:56:10,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cb6cd7f4b83e41159e5bd6e55278bffd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cb6cd7f4b83e41159e5bd6e55278bffd 2024-11-25T10:56:10,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/e400c9c2aad947688194b40ae7356eff to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/e400c9c2aad947688194b40ae7356eff 2024-11-25T10:56:10,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2e657c464d0349359380d419d7f1a47e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2e657c464d0349359380d419d7f1a47e 2024-11-25T10:56:10,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/b3c0865bab4644b0baafc62fa809cbf8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/b3c0865bab4644b0baafc62fa809cbf8 2024-11-25T10:56:10,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5f8a3ecfbda042c4a87d9ba0182f2c2b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5f8a3ecfbda042c4a87d9ba0182f2c2b 2024-11-25T10:56:10,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea4b6f8af55e4fe484257bcef2489b8e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/ea4b6f8af55e4fe484257bcef2489b8e 2024-11-25T10:56:10,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2066118267354f87806f56786b53092e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2066118267354f87806f56786b53092e 2024-11-25T10:56:10,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/8ad99027aa644846b8a1eee42692f1a2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/8ad99027aa644846b8a1eee42692f1a2 2024-11-25T10:56:10,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/1b091ef19fdd47e5bc5ae712c7465fcd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/1b091ef19fdd47e5bc5ae712c7465fcd 2024-11-25T10:56:10,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5697620dd2e04f14b62ab5e10f837813 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/5697620dd2e04f14b62ab5e10f837813 2024-11-25T10:56:10,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/62ef2c4fc00f487298df9f22b74d4e89 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/62ef2c4fc00f487298df9f22b74d4e89 2024-11-25T10:56:10,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/592db97099744bfd9ee7bef7388523c0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/592db97099744bfd9ee7bef7388523c0 2024-11-25T10:56:10,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/018aaf1c23274c98b865b7bb5c2cc0a3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/018aaf1c23274c98b865b7bb5c2cc0a3 2024-11-25T10:56:10,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/79827b9e9ecc4f16a086a34f697f1ada to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/79827b9e9ecc4f16a086a34f697f1ada 2024-11-25T10:56:10,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a9e80bfdc79044abad9f542fb6b43132 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a9e80bfdc79044abad9f542fb6b43132 2024-11-25T10:56:10,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2efaebf6743c45c9977a674fa89fcf1a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/2efaebf6743c45c9977a674fa89fcf1a 2024-11-25T10:56:10,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/da2349044dcc4da78985c6803281f221 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/da2349044dcc4da78985c6803281f221 2024-11-25T10:56:10,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/17cc458cafc14801ab5a24753c10884b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/17cc458cafc14801ab5a24753c10884b 2024-11-25T10:56:10,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a4a5321f532d4822b6e16c38601e811c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/a4a5321f532d4822b6e16c38601e811c 2024-11-25T10:56:10,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/22404d4fa2b84233a4935f19cc4e675b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0a4636b762734b4aa5b033771dae369c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ab5cd94e4fe14cfc9b5f44c7bd7d1ac3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/6339dda1cda547b5894bfa4e5d825270, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/986ffbd1dace4daba7d6d1d6ba8791a8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/bf2506933c3f47159c0d0acb38927473, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4dd3138a8b7a40debddbdbaa25f7faef, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/aaa0420d1f2d49129ce49d5698ede242, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/96ae99068aca454f854ccde6ebf88c20, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/04937fb736434b6eb22b7a2d503ce469, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/a78af616eaef4545b9e60d7b627ee571, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/29bf23fbaaaa40fe93c7878c2cab11b4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/592ca637f73b41a9ab6974e4e2025293, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0029d6f4f26145408cc6e7d2bc897d03, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/87a0da27fc8d4935af507ca15a062360, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ff7596668adb4b85a06f4675228e3350, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0092184dbd4740d78d51593a97c588bd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/388275fc560645258b7f8192d31ac1f6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/86648ad71cce463fa443bcba1ea4f97f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/9e5f9e83add141e3bcf4de579205a10c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/31f190a5111143b19cd0df125ea7b789] to archive 2024-11-25T10:56:10,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:56:10,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/22404d4fa2b84233a4935f19cc4e675b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/22404d4fa2b84233a4935f19cc4e675b 2024-11-25T10:56:10,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0a4636b762734b4aa5b033771dae369c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0a4636b762734b4aa5b033771dae369c 2024-11-25T10:56:10,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ab5cd94e4fe14cfc9b5f44c7bd7d1ac3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ab5cd94e4fe14cfc9b5f44c7bd7d1ac3 2024-11-25T10:56:10,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/6339dda1cda547b5894bfa4e5d825270 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/6339dda1cda547b5894bfa4e5d825270 2024-11-25T10:56:10,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/986ffbd1dace4daba7d6d1d6ba8791a8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/986ffbd1dace4daba7d6d1d6ba8791a8 2024-11-25T10:56:10,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/bf2506933c3f47159c0d0acb38927473 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/bf2506933c3f47159c0d0acb38927473 2024-11-25T10:56:10,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4dd3138a8b7a40debddbdbaa25f7faef to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4dd3138a8b7a40debddbdbaa25f7faef 2024-11-25T10:56:10,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/aaa0420d1f2d49129ce49d5698ede242 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/aaa0420d1f2d49129ce49d5698ede242 2024-11-25T10:56:10,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/96ae99068aca454f854ccde6ebf88c20 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/96ae99068aca454f854ccde6ebf88c20 2024-11-25T10:56:10,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/04937fb736434b6eb22b7a2d503ce469 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/04937fb736434b6eb22b7a2d503ce469 2024-11-25T10:56:10,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/a78af616eaef4545b9e60d7b627ee571 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/a78af616eaef4545b9e60d7b627ee571 2024-11-25T10:56:10,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/29bf23fbaaaa40fe93c7878c2cab11b4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/29bf23fbaaaa40fe93c7878c2cab11b4 2024-11-25T10:56:10,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/592ca637f73b41a9ab6974e4e2025293 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/592ca637f73b41a9ab6974e4e2025293 2024-11-25T10:56:10,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0029d6f4f26145408cc6e7d2bc897d03 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0029d6f4f26145408cc6e7d2bc897d03 2024-11-25T10:56:10,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/87a0da27fc8d4935af507ca15a062360 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/87a0da27fc8d4935af507ca15a062360 2024-11-25T10:56:10,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ff7596668adb4b85a06f4675228e3350 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/ff7596668adb4b85a06f4675228e3350 2024-11-25T10:56:10,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0092184dbd4740d78d51593a97c588bd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/0092184dbd4740d78d51593a97c588bd 2024-11-25T10:56:10,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/388275fc560645258b7f8192d31ac1f6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/388275fc560645258b7f8192d31ac1f6 2024-11-25T10:56:10,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/86648ad71cce463fa443bcba1ea4f97f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/86648ad71cce463fa443bcba1ea4f97f 2024-11-25T10:56:10,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/9e5f9e83add141e3bcf4de579205a10c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/9e5f9e83add141e3bcf4de579205a10c 2024-11-25T10:56:10,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/31f190a5111143b19cd0df125ea7b789 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/31f190a5111143b19cd0df125ea7b789 2024-11-25T10:56:10,710 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/recovered.edits/318.seqid, newMaxSeqId=318, maxSeqId=4 2024-11-25T10:56:10,713 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. 2024-11-25T10:56:10,713 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 78f10d0299f2958c87d34c58cfcb3253: Waiting for close lock at 1732532169109Running coprocessor pre-close hooks at 1732532169109Disabling compacts and flushes for region at 1732532169109Disabling writes for close at 1732532169205 (+96 ms)Obtaining lock to block concurrent updates at 1732532169206 (+1 ms)Preparing flush snapshotting stores in 78f10d0299f2958c87d34c58cfcb3253 at 1732532169206Finished memstore snapshotting TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732532169206Flushing stores of TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253. at 1732532169208 (+2 ms)Flushing 78f10d0299f2958c87d34c58cfcb3253/A: creating writer at 1732532169208Flushing mob file 78f10d0299f2958c87d34c58cfcb3253/A: appending metadata at 1732532169221 (+13 ms)Flushing mob file 78f10d0299f2958c87d34c58cfcb3253/A: closing flushed file at 1732532169221Flushing 78f10d0299f2958c87d34c58cfcb3253/A: appending metadata at 1732532169659 (+438 ms)Flushing 78f10d0299f2958c87d34c58cfcb3253/A: closing flushed file at 1732532169660 (+1 ms)Flushing 78f10d0299f2958c87d34c58cfcb3253/B: creating writer at 1732532169729 (+69 ms)Flushing 78f10d0299f2958c87d34c58cfcb3253/B: appending metadata at 1732532169746 (+17 ms)Flushing 78f10d0299f2958c87d34c58cfcb3253/B: closing flushed file at 1732532169746Flushing 78f10d0299f2958c87d34c58cfcb3253/C: creating writer at 1732532170197 (+451 ms)Flushing 78f10d0299f2958c87d34c58cfcb3253/C: appending metadata at 1732532170201 (+4 ms)Flushing 78f10d0299f2958c87d34c58cfcb3253/C: closing flushed file at 1732532170201Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@146f0953: reopening flushed file at 1732532170241 (+40 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@510ea5d3: reopening flushed file at 1732532170248 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f4c6ef2: reopening flushed file at 1732532170255 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 78f10d0299f2958c87d34c58cfcb3253 in 1058ms, sequenceid=315, compaction requested=false at 1732532170263 (+8 ms)Writing region close event to WAL at 1732532170677 (+414 ms)Running coprocessor post-close hooks at 1732532170713 (+36 ms)Closed at 1732532170713 2024-11-25T10:56:10,725 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:10,733 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=78f10d0299f2958c87d34c58cfcb3253, regionState=CLOSED 2024-11-25T10:56:10,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:56:10,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-25T10:56:10,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 78f10d0299f2958c87d34c58cfcb3253, server=6bfc37bad115,44671,1732532110409 in 1.8130 sec 2024-11-25T10:56:10,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-25T10:56:10,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78f10d0299f2958c87d34c58cfcb3253, UNASSIGN in 1.8330 sec 2024-11-25T10:56:10,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-25T10:56:10,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8440 sec 2024-11-25T10:56:10,765 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532170764"}]},"ts":"1732532170764"} 2024-11-25T10:56:10,767 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T10:56:10,767 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T10:56:10,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8860 sec 2024-11-25T10:56:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T10:56:11,065 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:11,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T10:56:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:11,089 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T10:56:11,091 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:11,107 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,125 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/recovered.edits] 2024-11-25T10:56:11,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/0f3246730a944b96a5f383ff8bc9e3a5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/0f3246730a944b96a5f383ff8bc9e3a5 2024-11-25T10:56:11,157 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/9d47f6d068154587879fb5a438cfae57 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/A/9d47f6d068154587879fb5a438cfae57 2024-11-25T10:56:11,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/6f90ad05d8e24d328f0f34bfd0f321ae to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/6f90ad05d8e24d328f0f34bfd0f321ae 2024-11-25T10:56:11,174 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cd6003cdd2f9436b851264f13cdbf773 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/B/cd6003cdd2f9436b851264f13cdbf773 2024-11-25T10:56:11,178 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4e8c90874fdf4996bf9855c94266b1d6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/4e8c90874fdf4996bf9855c94266b1d6 2024-11-25T10:56:11,179 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/d23bdff3a5364fa7b50283ba73d44c7b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/C/d23bdff3a5364fa7b50283ba73d44c7b 2024-11-25T10:56:11,183 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/recovered.edits/318.seqid to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253/recovered.edits/318.seqid 2024-11-25T10:56:11,184 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,184 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T10:56:11,185 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T10:56:11,186 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T10:56:11,195 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125066492813b5e477fb4614920c95a7d07_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125066492813b5e477fb4614920c95a7d07_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T10:56:11,197 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252be06c6940114690a8ae45de52205507_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252be06c6940114690a8ae45de52205507_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,199 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125687428b784c540f8b291fa3608a5efbc_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125687428b784c540f8b291fa3608a5efbc_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,201 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112574cafe5c3d354d13a1eb98827dcc2165_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112574cafe5c3d354d13a1eb98827dcc2165_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,202 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112579a33a8c91324b87b2fa51604a92327a_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112579a33a8c91324b87b2fa51604a92327a_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,204 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257dc900fe59f04bb1b976277b586f97a1_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257dc900fe59f04bb1b976277b586f97a1_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,206 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125818bb5be563546f48763688f3b1bd116_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125818bb5be563546f48763688f3b1bd116_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,208 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259372a7bd6351481ea92f8c31b6380048_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259372a7bd6351481ea92f8c31b6380048_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,210 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259bd85d6191924d388b4085e3a793cc9c_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259bd85d6191924d388b4085e3a793cc9c_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,212 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b4c9abd53bc5430c8250bde72647ceca_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b4c9abd53bc5430c8250bde72647ceca_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,214 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bdb5f5672fb043ecb0c9867aa0c3b8b6_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bdb5f5672fb043ecb0c9867aa0c3b8b6_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,216 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e174abfb9d52451da35ca3d1f0adeb56_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e174abfb9d52451da35ca3d1f0adeb56_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,218 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e9cdd12f9cd64c95b30d4882a84ab68e_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e9cdd12f9cd64c95b30d4882a84ab68e_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,220 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ef324b8746f24ae49f62f982a83e15f9_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ef324b8746f24ae49f62f982a83e15f9_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,222 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f0a77b4e502b41c085c0f57c8dfc5b79_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f0a77b4e502b41c085c0f57c8dfc5b79_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,223 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f42e35bd0b68469eac129ed1190aa9b5_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f42e35bd0b68469eac129ed1190aa9b5_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,225 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125fb54397504b646329b7935bfda0bc202_78f10d0299f2958c87d34c58cfcb3253 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125fb54397504b646329b7935bfda0bc202_78f10d0299f2958c87d34c58cfcb3253 2024-11-25T10:56:11,226 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T10:56:11,230 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:11,235 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T10:56:11,248 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T10:56:11,253 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:11,254 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T10:56:11,254 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732532171254"}]},"ts":"9223372036854775807"} 2024-11-25T10:56:11,265 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T10:56:11,265 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 78f10d0299f2958c87d34c58cfcb3253, NAME => 'TestAcidGuarantees,,1732532143764.78f10d0299f2958c87d34c58cfcb3253.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T10:56:11,265 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T10:56:11,266 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732532171265"}]},"ts":"9223372036854775807"} 2024-11-25T10:56:11,277 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T10:56:11,289 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:11,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 223 msec 2024-11-25T10:56:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T10:56:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T10:56:11,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:11,429 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 246) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1892856258_22 at /127.0.0.1:36302 [Waiting for operation #171] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-242236033_22 at /127.0.0.1:45268 [Waiting for operation #205] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1892856258_22 at /127.0.0.1:51876 [Waiting for operation #101] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1396 (was 1231) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1753 (was 1403) - AvailableMemoryMB LEAK? - 2024-11-25T10:56:11,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=1396, ProcessCount=11, AvailableMemoryMB=1751 2024-11-25T10:56:11,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:56:11,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:56:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:11,445 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T10:56:11,445 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:11,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-25T10:56:11,446 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T10:56:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-25T10:56:11,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742019_1195 (size=1061) 2024-11-25T10:56:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-25T10:56:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-25T10:56:11,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:56:11,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742020_1196 (size=53) 2024-11-25T10:56:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-25T10:56:12,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:56:12,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d938fc35c4eb636e5192d194067fd994, disabling compactions & flushes 2024-11-25T10:56:12,346 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. after waiting 0 ms 2024-11-25T10:56:12,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,346 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d938fc35c4eb636e5192d194067fd994: Waiting for close lock at 1732532172346Disabling compacts and flushes for region at 1732532172346Disabling writes for close at 1732532172346Writing region close event to WAL at 1732532172346Closed at 1732532172346 2024-11-25T10:56:12,348 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T10:56:12,348 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732532172348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532172348"}]},"ts":"1732532172348"} 2024-11-25T10:56:12,353 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T10:56:12,354 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T10:56:12,356 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532172355"}]},"ts":"1732532172355"} 2024-11-25T10:56:12,366 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T10:56:12,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, ASSIGN}] 2024-11-25T10:56:12,373 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, ASSIGN 2024-11-25T10:56:12,383 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, ASSIGN; state=OFFLINE, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=false 2024-11-25T10:56:12,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=d938fc35c4eb636e5192d194067fd994, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:12,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, ASSIGN because future has completed 2024-11-25T10:56:12,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:56:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-25T10:56:12,730 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,730 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:56:12,730 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,731 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:56:12,731 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,731 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,753 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,765 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:12,765 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d938fc35c4eb636e5192d194067fd994 columnFamilyName A 2024-11-25T10:56:12,766 DEBUG [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:12,769 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.HStore(327): Store=d938fc35c4eb636e5192d194067fd994/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:12,769 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,772 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:12,772 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d938fc35c4eb636e5192d194067fd994 columnFamilyName B 2024-11-25T10:56:12,772 DEBUG [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:12,773 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.HStore(327): Store=d938fc35c4eb636e5192d194067fd994/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:12,773 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,775 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:12,776 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d938fc35c4eb636e5192d194067fd994 columnFamilyName C 2024-11-25T10:56:12,776 DEBUG [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:12,776 INFO [StoreOpener-d938fc35c4eb636e5192d194067fd994-1 {}] regionserver.HStore(327): Store=d938fc35c4eb636e5192d194067fd994/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:12,777 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,777 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,778 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,778 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,785 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,785 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,785 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:56:12,787 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,800 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:56:12,801 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened d938fc35c4eb636e5192d194067fd994; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59888667, jitterRate=-0.10758931934833527}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:56:12,801 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:12,802 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for d938fc35c4eb636e5192d194067fd994: Running coprocessor pre-open hook at 1732532172731Writing region info on filesystem at 1732532172731Initializing all the Stores at 1732532172743 (+12 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532172743Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532172744 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532172744Cleaning up temporary data from old regions at 1732532172785 (+41 ms)Running coprocessor post-open hooks at 1732532172801 (+16 ms)Region opened successfully at 1732532172802 (+1 ms) 2024-11-25T10:56:12,807 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., pid=55, masterSystemTime=1732532172712 2024-11-25T10:56:12,814 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,814 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:12,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=d938fc35c4eb636e5192d194067fd994, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:12,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:56:12,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-25T10:56:12,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 in 289 msec 2024-11-25T10:56:12,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-25T10:56:12,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, ASSIGN in 465 msec 2024-11-25T10:56:12,841 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T10:56:12,841 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532172841"}]},"ts":"1732532172841"} 2024-11-25T10:56:12,843 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T10:56:12,844 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T10:56:12,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.4010 sec 2024-11-25T10:56:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-25T10:56:13,607 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:13,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78af2e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,610 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378ac1aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,614 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2848bfac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,615 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T10:56:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39013,1732532109078 2024-11-25T10:56:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b1927d4 2024-11-25T10:56:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T10:56:13,632 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T10:56:13,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,636 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f690496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2174d36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,645 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ecf78e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,647 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66586766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,649 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4843dea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,651 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b91bae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d37e917, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f62c95f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71178041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a79774c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209b8ca1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,687 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a50f94d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,694 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1d89c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ba1c66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6393382c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,701 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b855c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,709 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c02713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a995906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:13,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:13,722 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14491f74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:13,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@726b7c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:13,742 DEBUG [Thread-948 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,744 DEBUG [Thread-953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,745 DEBUG [Thread-955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,746 DEBUG [Thread-957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,747 DEBUG [Thread-957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,748 DEBUG [Thread-961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,750 DEBUG [Thread-961 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,751 DEBUG [Thread-961 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,753 DEBUG [Thread-944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,765 DEBUG [Thread-942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,768 DEBUG [Thread-946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,770 DEBUG [Thread-946 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,771 DEBUG [Thread-946 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:13,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:56:13,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:13,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:13,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:13,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:13,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:13,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:13,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,797 DEBUG [Thread-950 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,798 DEBUG [Thread-950 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,798 DEBUG [Thread-950 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,803 DEBUG [Thread-950 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:13,807 DEBUG [Thread-959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-25T10:56:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T10:56:13,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:56:13,817 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:13,819 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:13,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:13,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/85a1d960b08742d0b00812ee5ba58a86 is 50, key is test_row_0/A:col10/1732532173777/Put/seqid=0 2024-11-25T10:56:13,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532173815, max timestamp=1732532173840Num uniques 859; 2024-11-25T10:56:13,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:13,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532173815, max timestamp=1732532173840Num uniques 859; 2024-11-25T10:56:13,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:13,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532173815, max timestamp=1732532173840Num uniques 859; 2024-11-25T10:56:13,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:13,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742021_1197 (size=9657) 2024-11-25T10:56:13,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/85a1d960b08742d0b00812ee5ba58a86 2024-11-25T10:56:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T10:56:13,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532233914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532233914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532233915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:13,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:13,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/05ff2790000b4cd2887a3e132a18b0f1 is 50, key is test_row_0/B:col10/1732532173777/Put/seqid=0 2024-11-25T10:56:13,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532233936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:13,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532233935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:13,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742022_1198 (size=9657) 2024-11-25T10:56:13,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T10:56:13,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:13,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:13,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:13,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532234055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532234055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532234055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532234063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532234074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T10:56:14,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T10:56:14,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:14,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:14,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:14,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532234265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532234267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532234277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532234283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532234283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T10:56:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:14,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/05ff2790000b4cd2887a3e132a18b0f1 2024-11-25T10:56:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T10:56:14,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/17ab1a97868f441ebaf5cf2a9a618eb2 is 50, key is test_row_0/C:col10/1732532173777/Put/seqid=0 2024-11-25T10:56:14,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742023_1199 (size=9657) 2024-11-25T10:56:14,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T10:56:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:14,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:14,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/17ab1a97868f441ebaf5cf2a9a618eb2 2024-11-25T10:56:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/85a1d960b08742d0b00812ee5ba58a86 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/85a1d960b08742d0b00812ee5ba58a86 2024-11-25T10:56:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532234573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/85a1d960b08742d0b00812ee5ba58a86, entries=100, sequenceid=12, filesize=9.4 K 2024-11-25T10:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532234584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/05ff2790000b4cd2887a3e132a18b0f1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/05ff2790000b4cd2887a3e132a18b0f1 2024-11-25T10:56:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532234597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532234597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532234606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:14,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/05ff2790000b4cd2887a3e132a18b0f1, entries=100, sequenceid=12, filesize=9.4 K 2024-11-25T10:56:14,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/17ab1a97868f441ebaf5cf2a9a618eb2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/17ab1a97868f441ebaf5cf2a9a618eb2 2024-11-25T10:56:14,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/17ab1a97868f441ebaf5cf2a9a618eb2, entries=100, sequenceid=12, filesize=9.4 K 2024-11-25T10:56:14,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d938fc35c4eb636e5192d194067fd994 in 864ms, sequenceid=12, compaction requested=false 2024-11-25T10:56:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:14,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-25T10:56:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:14,651 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-25T10:56:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:14,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0b2423c49a9740d9b1aac8cf0ba3165c is 50, key is test_row_0/A:col10/1732532173930/Put/seqid=0 2024-11-25T10:56:14,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742024_1200 (size=12001) 2024-11-25T10:56:14,700 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0b2423c49a9740d9b1aac8cf0ba3165c 2024-11-25T10:56:14,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/bbec0da6633c47498ebaf1e87d4f706f is 50, key is test_row_0/B:col10/1732532173930/Put/seqid=0 2024-11-25T10:56:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742025_1201 (size=12001) 2024-11-25T10:56:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T10:56:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:15,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:15,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532235133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532235134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532235134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532235134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532235142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,190 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/bbec0da6633c47498ebaf1e87d4f706f 2024-11-25T10:56:15,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/839d9200836a4d109e4b9adad72f7da0 is 50, key is test_row_0/C:col10/1732532173930/Put/seqid=0 2024-11-25T10:56:15,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532235243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532235254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532235255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532235254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532235258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742026_1202 (size=12001) 2024-11-25T10:56:15,282 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/839d9200836a4d109e4b9adad72f7da0 2024-11-25T10:56:15,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0b2423c49a9740d9b1aac8cf0ba3165c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0b2423c49a9740d9b1aac8cf0ba3165c 2024-11-25T10:56:15,328 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0b2423c49a9740d9b1aac8cf0ba3165c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T10:56:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/bbec0da6633c47498ebaf1e87d4f706f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/bbec0da6633c47498ebaf1e87d4f706f 2024-11-25T10:56:15,345 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/bbec0da6633c47498ebaf1e87d4f706f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T10:56:15,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/839d9200836a4d109e4b9adad72f7da0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/839d9200836a4d109e4b9adad72f7da0 2024-11-25T10:56:15,354 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/839d9200836a4d109e4b9adad72f7da0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T10:56:15,356 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for d938fc35c4eb636e5192d194067fd994 in 704ms, sequenceid=49, compaction requested=false 2024-11-25T10:56:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-25T10:56:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-25T10:56:15,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-25T10:56:15,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5430 sec 2024-11-25T10:56:15,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.5640 sec 2024-11-25T10:56:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:15,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T10:56:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:15,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/764b6ec81093483c95ab904136827094 is 50, key is test_row_0/A:col10/1732532175131/Put/seqid=0 2024-11-25T10:56:15,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742027_1203 (size=14341) 2024-11-25T10:56:15,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/764b6ec81093483c95ab904136827094 2024-11-25T10:56:15,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532235526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532175490, max timestamp=1732532175552Num uniques 859; 2024-11-25T10:56:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:15,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732532175520, max timestamp=1732532175552Num uniques 859; 2024-11-25T10:56:15,563 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T10:56:15,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732532175520, max timestamp=1732532175552Num uniques 859; 2024-11-25T10:56:15,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T10:56:15,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a631d75de6624a98bf46aecd5e266fe6 is 50, key is test_row_0/B:col10/1732532175131/Put/seqid=0 2024-11-25T10:56:15,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742028_1204 (size=12001) 2024-11-25T10:56:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532235630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532235634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532235637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532235639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532235664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532235745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532235749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532235755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532235756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532235883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T10:56:15,969 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:15,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-25T10:56:15,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532235973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T10:56:15,984 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,985 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:15,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532235984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532235984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:15,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532235984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a631d75de6624a98bf46aecd5e266fe6 2024-11-25T10:56:16,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/e0a9961b776141719f2339c97b22dec6 is 50, key is test_row_0/C:col10/1732532175131/Put/seqid=0 2024-11-25T10:56:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T10:56:16,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742029_1205 (size=12001) 2024-11-25T10:56:16,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/e0a9961b776141719f2339c97b22dec6 2024-11-25T10:56:16,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T10:56:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:16,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:16,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:16,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/764b6ec81093483c95ab904136827094 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/764b6ec81093483c95ab904136827094 2024-11-25T10:56:16,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532236204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/764b6ec81093483c95ab904136827094, entries=200, sequenceid=61, filesize=14.0 K 2024-11-25T10:56:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a631d75de6624a98bf46aecd5e266fe6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a631d75de6624a98bf46aecd5e266fe6 2024-11-25T10:56:16,262 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T10:56:16,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a631d75de6624a98bf46aecd5e266fe6, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T10:56:16,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/e0a9961b776141719f2339c97b22dec6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e0a9961b776141719f2339c97b22dec6 2024-11-25T10:56:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T10:56:16,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e0a9961b776141719f2339c97b22dec6, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T10:56:16,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=122.21 KB/125140 for d938fc35c4eb636e5192d194067fd994 in 841ms, sequenceid=61, compaction requested=true 2024-11-25T10:56:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:56:16,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-25T10:56:16,304 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:16,304 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:16,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:16,305 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=122.21 KB heapSize=348.39 KB 2024-11-25T10:56:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:16,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:16,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:16,317 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:16,317 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:16,317 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:16,317 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:16,317 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:16,317 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:16,317 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/85a1d960b08742d0b00812ee5ba58a86, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0b2423c49a9740d9b1aac8cf0ba3165c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/764b6ec81093483c95ab904136827094] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.2 K 2024-11-25T10:56:16,317 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/17ab1a97868f441ebaf5cf2a9a618eb2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/839d9200836a4d109e4b9adad72f7da0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e0a9961b776141719f2339c97b22dec6] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=32.9 K 2024-11-25T10:56:16,325 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ab1a97868f441ebaf5cf2a9a618eb2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732532173776 2024-11-25T10:56:16,328 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a1d960b08742d0b00812ee5ba58a86, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732532173776 2024-11-25T10:56:16,333 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2423c49a9740d9b1aac8cf0ba3165c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532173899 2024-11-25T10:56:16,333 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 839d9200836a4d109e4b9adad72f7da0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532173899 2024-11-25T10:56:16,338 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a9961b776141719f2339c97b22dec6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532175119 2024-11-25T10:56:16,338 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 764b6ec81093483c95ab904136827094, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532175119 2024-11-25T10:56:16,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/87921f7754f34600813f2589d81d760c is 50, key is test_row_0/A:col10/1732532175625/Put/seqid=0 2024-11-25T10:56:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532236368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532236372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532236372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532236374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742030_1206 (size=12001) 2024-11-25T10:56:16,425 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:16,426 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/83f543e6ba264ea499da75f829896744 is 50, key is test_row_0/A:col10/1732532175131/Put/seqid=0 2024-11-25T10:56:16,429 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:16,430 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/15d07a1a797b4dcdb946bbd08195e043 is 50, key is test_row_0/C:col10/1732532175131/Put/seqid=0 2024-11-25T10:56:16,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742031_1207 (size=12104) 2024-11-25T10:56:16,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742032_1208 (size=12104) 2024-11-25T10:56:16,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532236504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532236505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532236505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532236505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,543 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/15d07a1a797b4dcdb946bbd08195e043 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/15d07a1a797b4dcdb946bbd08195e043 2024-11-25T10:56:16,581 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 15d07a1a797b4dcdb946bbd08195e043(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:16,581 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:16,581 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532176301; duration=0sec 2024-11-25T10:56:16,581 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:16,581 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:16,581 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:16,589 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:16,590 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:16,590 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:16,590 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/05ff2790000b4cd2887a3e132a18b0f1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/bbec0da6633c47498ebaf1e87d4f706f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a631d75de6624a98bf46aecd5e266fe6] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=32.9 K 2024-11-25T10:56:16,590 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ff2790000b4cd2887a3e132a18b0f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732532173776 2024-11-25T10:56:16,591 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bbec0da6633c47498ebaf1e87d4f706f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732532173899 2024-11-25T10:56:16,591 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a631d75de6624a98bf46aecd5e266fe6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532175119 2024-11-25T10:56:16,611 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:16,612 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/e5b7e98312d2465da58375c2d2f359cf is 50, key is test_row_0/B:col10/1732532175131/Put/seqid=0 2024-11-25T10:56:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T10:56:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742033_1209 (size=12104) 2024-11-25T10:56:16,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532236717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532236724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532236724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532236725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532236740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:16,765 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/e5b7e98312d2465da58375c2d2f359cf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e5b7e98312d2465da58375c2d2f359cf 2024-11-25T10:56:16,793 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into e5b7e98312d2465da58375c2d2f359cf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:16,793 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:16,793 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532176301; duration=0sec 2024-11-25T10:56:16,793 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:16,794 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:16,821 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/87921f7754f34600813f2589d81d760c 2024-11-25T10:56:16,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/be40d3cfa74c42f690adfdb3d94a7e12 is 50, key is test_row_0/B:col10/1732532175625/Put/seqid=0 2024-11-25T10:56:16,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742034_1210 (size=12001) 2024-11-25T10:56:16,916 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/be40d3cfa74c42f690adfdb3d94a7e12 2024-11-25T10:56:16,925 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/83f543e6ba264ea499da75f829896744 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/83f543e6ba264ea499da75f829896744 2024-11-25T10:56:16,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/71eeac57e2a3492fb7e99947669626c9 is 50, key is test_row_0/C:col10/1732532175625/Put/seqid=0 2024-11-25T10:56:16,969 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 83f543e6ba264ea499da75f829896744(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:16,969 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:16,969 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532176301; duration=0sec 2024-11-25T10:56:16,969 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:16,969 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:17,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742035_1211 (size=12001) 2024-11-25T10:56:17,017 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/71eeac57e2a3492fb7e99947669626c9 2024-11-25T10:56:17,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532237044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532237045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532237055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/87921f7754f34600813f2589d81d760c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/87921f7754f34600813f2589d81d760c 2024-11-25T10:56:17,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532237064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T10:56:17,134 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/87921f7754f34600813f2589d81d760c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T10:56:17,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/be40d3cfa74c42f690adfdb3d94a7e12 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/be40d3cfa74c42f690adfdb3d94a7e12 2024-11-25T10:56:17,182 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/be40d3cfa74c42f690adfdb3d94a7e12, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T10:56:17,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/71eeac57e2a3492fb7e99947669626c9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/71eeac57e2a3492fb7e99947669626c9 2024-11-25T10:56:17,213 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/71eeac57e2a3492fb7e99947669626c9, entries=150, sequenceid=96, filesize=11.7 K 2024-11-25T10:56:17,216 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~122.21 KB/125140, heapSize ~348.34 KB/356704, currentSize=60.38 KB/61830 for d938fc35c4eb636e5192d194067fd994 in 912ms, sequenceid=96, compaction requested=false 2024-11-25T10:56:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-25T10:56:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-25T10:56:17,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-25T10:56:17,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2450 sec 2024-11-25T10:56:17,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.2890 sec 2024-11-25T10:56:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:17,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T10:56:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:17,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0ffbb8e1f8d14f47a4bc613d62c46f80 is 50, key is test_row_0/A:col10/1732532176370/Put/seqid=0 2024-11-25T10:56:17,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742036_1212 (size=12001) 2024-11-25T10:56:17,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532237656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532237656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532237672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532237677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532237734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,740 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532237783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532237783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532237796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532237797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532237994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532237998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532238003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532238005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0ffbb8e1f8d14f47a4bc613d62c46f80 2024-11-25T10:56:18,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/9c02192ec47949bd8f02d86d4620e2e5 is 50, key is test_row_0/B:col10/1732532176370/Put/seqid=0 2024-11-25T10:56:18,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742037_1213 (size=12001) 2024-11-25T10:56:18,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T10:56:18,146 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:18,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-25T10:56:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T10:56:18,148 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:18,149 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:18,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T10:56:18,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532238303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T10:56:18,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532238381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532238382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532238382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T10:56:18,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/9c02192ec47949bd8f02d86d4620e2e5 2024-11-25T10:56:18,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T10:56:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:18,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/f94d129172324fe5b0536b21c04442f8 is 50, key is test_row_0/C:col10/1732532176370/Put/seqid=0 2024-11-25T10:56:18,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742038_1214 (size=12001) 2024-11-25T10:56:18,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T10:56:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T10:56:18,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T10:56:18,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:18,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:18,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532238897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532238897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532238901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532238905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:19,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/f94d129172324fe5b0536b21c04442f8 2024-11-25T10:56:19,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T10:56:19,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:19,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:19,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:19,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:19,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:19,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0ffbb8e1f8d14f47a4bc613d62c46f80 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0ffbb8e1f8d14f47a4bc613d62c46f80 2024-11-25T10:56:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:19,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0ffbb8e1f8d14f47a4bc613d62c46f80, entries=150, sequenceid=113, filesize=11.7 K 2024-11-25T10:56:19,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/9c02192ec47949bd8f02d86d4620e2e5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9c02192ec47949bd8f02d86d4620e2e5 2024-11-25T10:56:19,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9c02192ec47949bd8f02d86d4620e2e5, entries=150, sequenceid=113, filesize=11.7 K 2024-11-25T10:56:19,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/f94d129172324fe5b0536b21c04442f8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f94d129172324fe5b0536b21c04442f8 2024-11-25T10:56:19,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f94d129172324fe5b0536b21c04442f8, entries=150, sequenceid=113, filesize=11.7 K 2024-11-25T10:56:19,119 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-25T10:56:19,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d938fc35c4eb636e5192d194067fd994 in 1542ms, sequenceid=113, compaction requested=true 2024-11-25T10:56:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:19,120 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:19,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:19,121 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:19,137 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:19,137 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:19,137 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:19,137 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:19,137 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:19,137 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e5b7e98312d2465da58375c2d2f359cf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/be40d3cfa74c42f690adfdb3d94a7e12, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9c02192ec47949bd8f02d86d4620e2e5] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.3 K 2024-11-25T10:56:19,138 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b7e98312d2465da58375c2d2f359cf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532175119 2024-11-25T10:56:19,139 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting be40d3cfa74c42f690adfdb3d94a7e12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732532175625 2024-11-25T10:56:19,139 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c02192ec47949bd8f02d86d4620e2e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732532176370 2024-11-25T10:56:19,140 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:19,140 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/83f543e6ba264ea499da75f829896744, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/87921f7754f34600813f2589d81d760c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0ffbb8e1f8d14f47a4bc613d62c46f80] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.3 K 2024-11-25T10:56:19,144 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f543e6ba264ea499da75f829896744, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532175119 2024-11-25T10:56:19,148 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 87921f7754f34600813f2589d81d760c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732532175625 2024-11-25T10:56:19,152 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ffbb8e1f8d14f47a4bc613d62c46f80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732532176370 2024-11-25T10:56:19,161 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:19,162 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/da78fab65ba845099cbdaf78431d8ffc is 50, key is test_row_0/B:col10/1732532176370/Put/seqid=0 2024-11-25T10:56:19,189 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:19,190 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/789fed84b586484980a506d6936154a0 is 50, key is test_row_0/A:col10/1732532176370/Put/seqid=0 2024-11-25T10:56:19,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T10:56:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:19,228 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:56:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:19,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742039_1215 (size=12207) 2024-11-25T10:56:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T10:56:19,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742040_1216 (size=12207) 2024-11-25T10:56:19,285 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/da78fab65ba845099cbdaf78431d8ffc as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/da78fab65ba845099cbdaf78431d8ffc 2024-11-25T10:56:19,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/189c3cbfb03d4239ad77ae4ee8529aa9 is 50, key is test_row_0/A:col10/1732532177649/Put/seqid=0 2024-11-25T10:56:19,302 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/789fed84b586484980a506d6936154a0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/789fed84b586484980a506d6936154a0 2024-11-25T10:56:19,325 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into da78fab65ba845099cbdaf78431d8ffc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:19,325 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:19,325 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532179120; duration=0sec 2024-11-25T10:56:19,325 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:19,325 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:19,325 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:19,326 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 789fed84b586484980a506d6936154a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:19,326 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:19,326 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532179120; duration=0sec 2024-11-25T10:56:19,326 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:19,326 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:19,327 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:19,327 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:19,327 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:19,327 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/15d07a1a797b4dcdb946bbd08195e043, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/71eeac57e2a3492fb7e99947669626c9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f94d129172324fe5b0536b21c04442f8] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.3 K 2024-11-25T10:56:19,328 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d07a1a797b4dcdb946bbd08195e043, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532175119 2024-11-25T10:56:19,328 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71eeac57e2a3492fb7e99947669626c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732532175625 2024-11-25T10:56:19,329 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f94d129172324fe5b0536b21c04442f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732532176370 2024-11-25T10:56:19,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742041_1217 (size=12051) 2024-11-25T10:56:19,365 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:19,366 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/212b63a7aff14a8293639c690b8ba716 is 50, key is test_row_0/C:col10/1732532176370/Put/seqid=0 2024-11-25T10:56:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742042_1218 (size=12207) 2024-11-25T10:56:19,438 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/212b63a7aff14a8293639c690b8ba716 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/212b63a7aff14a8293639c690b8ba716 2024-11-25T10:56:19,453 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 212b63a7aff14a8293639c690b8ba716(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:19,453 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:19,453 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532179120; duration=0sec 2024-11-25T10:56:19,453 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:19,454 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:19,736 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/189c3cbfb03d4239ad77ae4ee8529aa9 2024-11-25T10:56:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/291072b6a1114b488115a8011aa82d4a is 50, key is test_row_0/B:col10/1732532177649/Put/seqid=0 2024-11-25T10:56:19,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:19,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742043_1219 (size=12051) 2024-11-25T10:56:19,810 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/291072b6a1114b488115a8011aa82d4a 2024-11-25T10:56:19,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/66bea01a666a4923b039748fde15c7ae is 50, key is test_row_0/C:col10/1732532177649/Put/seqid=0 2024-11-25T10:56:19,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742044_1220 (size=12051) 2024-11-25T10:56:19,913 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/66bea01a666a4923b039748fde15c7ae 2024-11-25T10:56:19,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/189c3cbfb03d4239ad77ae4ee8529aa9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/189c3cbfb03d4239ad77ae4ee8529aa9 2024-11-25T10:56:19,933 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/189c3cbfb03d4239ad77ae4ee8529aa9, entries=150, sequenceid=131, filesize=11.8 K 2024-11-25T10:56:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/291072b6a1114b488115a8011aa82d4a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/291072b6a1114b488115a8011aa82d4a 2024-11-25T10:56:19,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532239925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:19,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532239925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:19,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532239926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:19,940 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:19,942 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_1' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:19,943 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/291072b6a1114b488115a8011aa82d4a, entries=150, sequenceid=131, filesize=11.8 K 2024-11-25T10:56:19,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/66bea01a666a4923b039748fde15c7ae as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/66bea01a666a4923b039748fde15c7ae 2024-11-25T10:56:19,949 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/66bea01a666a4923b039748fde15c7ae, entries=150, sequenceid=131, filesize=11.8 K 2024-11-25T10:56:19,951 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d938fc35c4eb636e5192d194067fd994 in 722ms, sequenceid=131, compaction requested=false 2024-11-25T10:56:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-25T10:56:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-25T10:56:19,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T10:56:19,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-25T10:56:19,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8100 sec 2024-11-25T10:56:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:19,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:56:19,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:19,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:19,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:19,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:19,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:19,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:19,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/5d09f5d479604062b2158afb95a450c2 is 50, key is test_row_0/A:col10/1732532179935/Put/seqid=0 2024-11-25T10:56:20,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742045_1221 (size=14541) 2024-11-25T10:56:20,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/5d09f5d479604062b2158afb95a450c2 2024-11-25T10:56:20,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0b2a0aecc10f411d8a994dcda17f905e is 50, key is test_row_0/B:col10/1732532179935/Put/seqid=0 2024-11-25T10:56:20,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532240056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532240062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532240074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742046_1222 (size=12151) 2024-11-25T10:56:20,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532240184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532240184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532240193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T10:56:20,293 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:20,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-25T10:56:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T10:56:20,310 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:20,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:20,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532240393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532240404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T10:56:20,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532240415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T10:56:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:20,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:20,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:20,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:20,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:20,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0b2a0aecc10f411d8a994dcda17f905e 2024-11-25T10:56:20,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/ff156b7b31d04833b52eb8f98b2e774a is 50, key is test_row_0/C:col10/1732532179935/Put/seqid=0 2024-11-25T10:56:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T10:56:20,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742047_1223 (size=12151) 2024-11-25T10:56:20,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/ff156b7b31d04833b52eb8f98b2e774a 2024-11-25T10:56:20,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T10:56:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:20,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:20,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:20,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/5d09f5d479604062b2158afb95a450c2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/5d09f5d479604062b2158afb95a450c2 2024-11-25T10:56:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532240717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532240717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/5d09f5d479604062b2158afb95a450c2, entries=200, sequenceid=151, filesize=14.2 K 2024-11-25T10:56:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0b2a0aecc10f411d8a994dcda17f905e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0b2a0aecc10f411d8a994dcda17f905e 2024-11-25T10:56:20,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532240723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:20,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0b2a0aecc10f411d8a994dcda17f905e, entries=150, sequenceid=151, filesize=11.9 K 2024-11-25T10:56:20,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/ff156b7b31d04833b52eb8f98b2e774a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ff156b7b31d04833b52eb8f98b2e774a 2024-11-25T10:56:20,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ff156b7b31d04833b52eb8f98b2e774a, entries=150, sequenceid=151, filesize=11.9 K 2024-11-25T10:56:20,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d938fc35c4eb636e5192d194067fd994 in 794ms, sequenceid=151, compaction requested=true 2024-11-25T10:56:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:20,764 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:20,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:20,764 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:20,770 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38799 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:20,770 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:20,770 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:20,770 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/789fed84b586484980a506d6936154a0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/189c3cbfb03d4239ad77ae4ee8529aa9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/5d09f5d479604062b2158afb95a450c2] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=37.9 K 2024-11-25T10:56:20,772 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 789fed84b586484980a506d6936154a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732532176370 2024-11-25T10:56:20,776 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:20,776 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 189c3cbfb03d4239ad77ae4ee8529aa9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732532177641 2024-11-25T10:56:20,776 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:20,777 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:20,777 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/da78fab65ba845099cbdaf78431d8ffc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/291072b6a1114b488115a8011aa82d4a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0b2a0aecc10f411d8a994dcda17f905e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.6 K 2024-11-25T10:56:20,784 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d09f5d479604062b2158afb95a450c2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732532179913 2024-11-25T10:56:20,788 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting da78fab65ba845099cbdaf78431d8ffc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732532176370 2024-11-25T10:56:20,793 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 291072b6a1114b488115a8011aa82d4a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732532177641 2024-11-25T10:56:20,800 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2a0aecc10f411d8a994dcda17f905e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732532179935 2024-11-25T10:56:20,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T10:56:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:20,811 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:56:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/7fc98d128a124485b0e016b0f183a2df is 50, key is test_row_0/A:col10/1732532180051/Put/seqid=0 2024-11-25T10:56:20,865 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#184 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:20,866 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/72855b411a084bc3ba36bc0f72135489 is 50, key is test_row_0/A:col10/1732532179935/Put/seqid=0 2024-11-25T10:56:20,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742049_1225 (size=12459) 2024-11-25T10:56:20,929 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#185 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:20,930 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/8111fb61eb184fb990fc9701689ae44a is 50, key is test_row_0/B:col10/1732532179935/Put/seqid=0 2024-11-25T10:56:20,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742048_1224 (size=12151) 2024-11-25T10:56:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T10:56:20,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742050_1226 (size=12459) 2024-11-25T10:56:21,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:21,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532241275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532241285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532241292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,336 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/7fc98d128a124485b0e016b0f183a2df 2024-11-25T10:56:21,343 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/72855b411a084bc3ba36bc0f72135489 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/72855b411a084bc3ba36bc0f72135489 2024-11-25T10:56:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/45c8da069a204d7eb5bcebb40c729f2f is 50, key is test_row_0/B:col10/1732532180051/Put/seqid=0 2024-11-25T10:56:21,351 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 72855b411a084bc3ba36bc0f72135489(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:21,351 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:21,351 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532180764; duration=0sec 2024-11-25T10:56:21,351 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:21,351 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:21,352 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:21,355 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:21,355 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:21,355 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:21,355 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/212b63a7aff14a8293639c690b8ba716, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/66bea01a666a4923b039748fde15c7ae, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ff156b7b31d04833b52eb8f98b2e774a] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.6 K 2024-11-25T10:56:21,356 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 212b63a7aff14a8293639c690b8ba716, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732532176370 2024-11-25T10:56:21,356 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 66bea01a666a4923b039748fde15c7ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732532177641 2024-11-25T10:56:21,363 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ff156b7b31d04833b52eb8f98b2e774a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732532179935 2024-11-25T10:56:21,399 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:21,400 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/5bbf3946060a45c59a241e7b8affbb75 is 50, key is test_row_0/C:col10/1732532179935/Put/seqid=0 2024-11-25T10:56:21,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742051_1227 (size=12151) 2024-11-25T10:56:21,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532241405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532241405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532241405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,431 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/8111fb61eb184fb990fc9701689ae44a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/8111fb61eb184fb990fc9701689ae44a 2024-11-25T10:56:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T10:56:21,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742052_1228 (size=12459) 2024-11-25T10:56:21,466 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into 8111fb61eb184fb990fc9701689ae44a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:21,466 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:21,466 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532180764; duration=0sec 2024-11-25T10:56:21,466 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:21,466 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:21,472 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/5bbf3946060a45c59a241e7b8affbb75 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/5bbf3946060a45c59a241e7b8affbb75 2024-11-25T10:56:21,486 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 5bbf3946060a45c59a241e7b8affbb75(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:21,486 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:21,486 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532180764; duration=0sec 2024-11-25T10:56:21,486 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:21,486 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:21,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532241623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532241623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532241625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,801 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/45c8da069a204d7eb5bcebb40c729f2f 2024-11-25T10:56:21,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/41ce56f31e194134a33d70e9b9638fb2 is 50, key is test_row_0/C:col10/1732532180051/Put/seqid=0 2024-11-25T10:56:21,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742053_1229 (size=12151) 2024-11-25T10:56:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532241936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532241936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532241945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532241965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532241965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:21,970 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_1' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:21,972 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,259 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/41ce56f31e194134a33d70e9b9638fb2 2024-11-25T10:56:22,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/7fc98d128a124485b0e016b0f183a2df as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7fc98d128a124485b0e016b0f183a2df 2024-11-25T10:56:22,283 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7fc98d128a124485b0e016b0f183a2df, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T10:56:22,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/45c8da069a204d7eb5bcebb40c729f2f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/45c8da069a204d7eb5bcebb40c729f2f 2024-11-25T10:56:22,292 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/45c8da069a204d7eb5bcebb40c729f2f, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T10:56:22,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/41ce56f31e194134a33d70e9b9638fb2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/41ce56f31e194134a33d70e9b9638fb2 2024-11-25T10:56:22,305 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/41ce56f31e194134a33d70e9b9638fb2, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T10:56:22,306 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d938fc35c4eb636e5192d194067fd994 in 1495ms, sequenceid=166, compaction requested=false 2024-11-25T10:56:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-25T10:56:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-25T10:56:22,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-25T10:56:22,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9860 sec 2024-11-25T10:56:22,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.0150 sec 2024-11-25T10:56:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T10:56:22,454 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:22,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:22,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:56:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-25T10:56:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T10:56:22,461 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:22,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:22,464 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:22,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:22,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/b09a83318fd94eb4b1298a1dbc9f10f4 is 50, key is test_row_0/A:col10/1732532182456/Put/seqid=0 2024-11-25T10:56:22,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742054_1230 (size=12151) 2024-11-25T10:56:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532242506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/b09a83318fd94eb4b1298a1dbc9f10f4 2024-11-25T10:56:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532242508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532242517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/6a569c10b22f4c24b9f9384101f53a7c is 50, key is test_row_0/B:col10/1732532182456/Put/seqid=0 2024-11-25T10:56:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T10:56:22,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742055_1231 (size=12151) 2024-11-25T10:56:22,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/6a569c10b22f4c24b9f9384101f53a7c 2024-11-25T10:56:22,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T10:56:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:22,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532242625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/d085b0c8f9654c20b80a5ffc1284835c is 50, key is test_row_0/C:col10/1732532182456/Put/seqid=0 2024-11-25T10:56:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532242628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532242634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742056_1232 (size=12151) 2024-11-25T10:56:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T10:56:22,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T10:56:22,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:22,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:22,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:22,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532242852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532242852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532242853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:22,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T10:56:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:22,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:22,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:23,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/d085b0c8f9654c20b80a5ffc1284835c 2024-11-25T10:56:23,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/b09a83318fd94eb4b1298a1dbc9f10f4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b09a83318fd94eb4b1298a1dbc9f10f4 2024-11-25T10:56:23,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b09a83318fd94eb4b1298a1dbc9f10f4, entries=150, sequenceid=188, filesize=11.9 K 2024-11-25T10:56:23,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/6a569c10b22f4c24b9f9384101f53a7c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/6a569c10b22f4c24b9f9384101f53a7c 2024-11-25T10:56:23,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/6a569c10b22f4c24b9f9384101f53a7c, entries=150, sequenceid=188, filesize=11.9 K 2024-11-25T10:56:23,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/d085b0c8f9654c20b80a5ffc1284835c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/d085b0c8f9654c20b80a5ffc1284835c 2024-11-25T10:56:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T10:56:23,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/d085b0c8f9654c20b80a5ffc1284835c, entries=150, sequenceid=188, filesize=11.9 K 2024-11-25T10:56:23,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d938fc35c4eb636e5192d194067fd994 in 640ms, sequenceid=188, compaction requested=true 2024-11-25T10:56:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:23,099 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:23,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:23,099 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:23,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T10:56:23,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,101 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:56:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:23,104 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:23,105 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:23,105 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,105 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/72855b411a084bc3ba36bc0f72135489, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7fc98d128a124485b0e016b0f183a2df, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b09a83318fd94eb4b1298a1dbc9f10f4] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.9 K 2024-11-25T10:56:23,106 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72855b411a084bc3ba36bc0f72135489, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732532179935 2024-11-25T10:56:23,106 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:23,106 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:23,106 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,106 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/8111fb61eb184fb990fc9701689ae44a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/45c8da069a204d7eb5bcebb40c729f2f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/6a569c10b22f4c24b9f9384101f53a7c] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.9 K 2024-11-25T10:56:23,108 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8111fb61eb184fb990fc9701689ae44a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732532179935 2024-11-25T10:56:23,108 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc98d128a124485b0e016b0f183a2df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732532180035 2024-11-25T10:56:23,109 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c8da069a204d7eb5bcebb40c729f2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732532180035 2024-11-25T10:56:23,109 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a569c10b22f4c24b9f9384101f53a7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732532181270 2024-11-25T10:56:23,109 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b09a83318fd94eb4b1298a1dbc9f10f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732532181270 2024-11-25T10:56:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1585d3284bac4149a1570d7ca51cd665 is 50, key is test_row_0/A:col10/1732532182512/Put/seqid=0 2024-11-25T10:56:23,123 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:23,124 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/4f2015fe3adc41789ae39f9a59db8c9f is 50, key is test_row_0/B:col10/1732532182456/Put/seqid=0 2024-11-25T10:56:23,124 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:23,125 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/24478366887745e9b47edfab0b1600b2 is 50, key is test_row_0/A:col10/1732532182456/Put/seqid=0 2024-11-25T10:56:23,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742057_1233 (size=12561) 2024-11-25T10:56:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:23,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742058_1234 (size=12561) 2024-11-25T10:56:23,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742059_1235 (size=12151) 2024-11-25T10:56:23,194 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1585d3284bac4149a1570d7ca51cd665 2024-11-25T10:56:23,205 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/24478366887745e9b47edfab0b1600b2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/24478366887745e9b47edfab0b1600b2 2024-11-25T10:56:23,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/2e40abeeb58342b093d96028778aaf16 is 50, key is test_row_0/B:col10/1732532182512/Put/seqid=0 2024-11-25T10:56:23,218 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 24478366887745e9b47edfab0b1600b2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:23,218 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:23,218 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532183099; duration=0sec 2024-11-25T10:56:23,218 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:23,218 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:23,218 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:23,225 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:23,225 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:23,226 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,226 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/5bbf3946060a45c59a241e7b8affbb75, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/41ce56f31e194134a33d70e9b9638fb2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/d085b0c8f9654c20b80a5ffc1284835c] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=35.9 K 2024-11-25T10:56:23,228 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bbf3946060a45c59a241e7b8affbb75, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732532179935 2024-11-25T10:56:23,236 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41ce56f31e194134a33d70e9b9638fb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732532180035 2024-11-25T10:56:23,236 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d085b0c8f9654c20b80a5ffc1284835c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732532181270 2024-11-25T10:56:23,249 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:23,250 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/69bd51f7d89046958c0c43084b336bbf is 50, key is test_row_0/C:col10/1732532182456/Put/seqid=0 2024-11-25T10:56:23,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742060_1236 (size=12151) 2024-11-25T10:56:23,273 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/2e40abeeb58342b093d96028778aaf16 2024-11-25T10:56:23,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532243263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532243267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532243282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742061_1237 (size=12561) 2024-11-25T10:56:23,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0e850dbf4dcc40039908921db8634395 is 50, key is test_row_0/C:col10/1732532182512/Put/seqid=0 2024-11-25T10:56:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532243396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532243397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532243403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742062_1238 (size=12151) 2024-11-25T10:56:23,420 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0e850dbf4dcc40039908921db8634395 2024-11-25T10:56:23,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1585d3284bac4149a1570d7ca51cd665 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1585d3284bac4149a1570d7ca51cd665 2024-11-25T10:56:23,471 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1585d3284bac4149a1570d7ca51cd665, entries=150, sequenceid=201, filesize=11.9 K 2024-11-25T10:56:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/2e40abeeb58342b093d96028778aaf16 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2e40abeeb58342b093d96028778aaf16 2024-11-25T10:56:23,481 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2e40abeeb58342b093d96028778aaf16, entries=150, sequenceid=201, filesize=11.9 K 2024-11-25T10:56:23,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0e850dbf4dcc40039908921db8634395 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0e850dbf4dcc40039908921db8634395 2024-11-25T10:56:23,490 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0e850dbf4dcc40039908921db8634395, entries=150, sequenceid=201, filesize=11.9 K 2024-11-25T10:56:23,500 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d938fc35c4eb636e5192d194067fd994 in 399ms, sequenceid=201, compaction requested=false 2024-11-25T10:56:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-25T10:56:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-25T10:56:23,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-25T10:56:23,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0450 sec 2024-11-25T10:56:23,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.0770 sec 2024-11-25T10:56:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T10:56:23,635 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:23,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-25T10:56:23,642 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T10:56:23,643 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:23,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:23,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532183256, max timestamp=1732532183645Num uniques 859; 2024-11-25T10:56:23,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-25T10:56:23,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532183256, max timestamp=1732532183645Num uniques 859; 2024-11-25T10:56:23,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:23,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:23,651 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/4f2015fe3adc41789ae39f9a59db8c9f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/4f2015fe3adc41789ae39f9a59db8c9f 2024-11-25T10:56:23,673 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into 4f2015fe3adc41789ae39f9a59db8c9f(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:23,673 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:23,673 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532183099; duration=0sec 2024-11-25T10:56:23,673 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:23,674 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:23,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/d9b8456837ec473290037dcec55c5e5d is 50, key is test_row_0/A:col10/1732532183645/Put/seqid=0 2024-11-25T10:56:23,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742063_1239 (size=14541) 2024-11-25T10:56:23,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/d9b8456837ec473290037dcec55c5e5d 2024-11-25T10:56:23,721 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/69bd51f7d89046958c0c43084b336bbf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/69bd51f7d89046958c0c43084b336bbf 2024-11-25T10:56:23,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a0be72a12fed45e486d82a8fb6a4e4d6 is 50, key is test_row_0/B:col10/1732532183645/Put/seqid=0 2024-11-25T10:56:23,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742064_1240 (size=12151) 2024-11-25T10:56:23,751 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 69bd51f7d89046958c0c43084b336bbf(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:23,751 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:23,751 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532183099; duration=0sec 2024-11-25T10:56:23,751 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:23,751 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T10:56:23,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532243764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532243766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532243768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T10:56:23,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:23,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532243885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532243885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532243884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:23,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T10:56:23,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T10:56:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:23,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:24,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532244106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532244107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532244109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T10:56:24,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:24,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:24,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:24,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:24,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:24,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a0be72a12fed45e486d82a8fb6a4e4d6 2024-11-25T10:56:24,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/bd9e6b54b6db454c8b77ad6167132697 is 50, key is test_row_0/C:col10/1732532183645/Put/seqid=0 2024-11-25T10:56:24,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742065_1241 (size=12151) 2024-11-25T10:56:24,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/bd9e6b54b6db454c8b77ad6167132697 2024-11-25T10:56:24,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/d9b8456837ec473290037dcec55c5e5d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/d9b8456837ec473290037dcec55c5e5d 2024-11-25T10:56:24,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/d9b8456837ec473290037dcec55c5e5d, entries=200, sequenceid=224, filesize=14.2 K 2024-11-25T10:56:24,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a0be72a12fed45e486d82a8fb6a4e4d6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a0be72a12fed45e486d82a8fb6a4e4d6 2024-11-25T10:56:24,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a0be72a12fed45e486d82a8fb6a4e4d6, entries=150, sequenceid=224, filesize=11.9 K 2024-11-25T10:56:24,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/bd9e6b54b6db454c8b77ad6167132697 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/bd9e6b54b6db454c8b77ad6167132697 2024-11-25T10:56:24,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/bd9e6b54b6db454c8b77ad6167132697, entries=150, sequenceid=224, filesize=11.9 K 2024-11-25T10:56:24,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~194.20 KB/198864, currentSize=107.34 KB/109920 for d938fc35c4eb636e5192d194067fd994 in 581ms, sequenceid=224, compaction requested=true 2024-11-25T10:56:24,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:24,229 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:24,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:24,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:24,230 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:24,231 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:24,231 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:24,231 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:24,231 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/24478366887745e9b47edfab0b1600b2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1585d3284bac4149a1570d7ca51cd665, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/d9b8456837ec473290037dcec55c5e5d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=38.3 K 2024-11-25T10:56:24,231 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:24,232 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:24,232 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:24,232 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/4f2015fe3adc41789ae39f9a59db8c9f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2e40abeeb58342b093d96028778aaf16, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a0be72a12fed45e486d82a8fb6a4e4d6] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.0 K 2024-11-25T10:56:24,232 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24478366887745e9b47edfab0b1600b2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732532181270 2024-11-25T10:56:24,232 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2015fe3adc41789ae39f9a59db8c9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732532181270 2024-11-25T10:56:24,233 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1585d3284bac4149a1570d7ca51cd665, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732532182468 2024-11-25T10:56:24,233 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e40abeeb58342b093d96028778aaf16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732532182468 2024-11-25T10:56:24,233 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9b8456837ec473290037dcec55c5e5d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532183278 2024-11-25T10:56:24,233 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a0be72a12fed45e486d82a8fb6a4e4d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532183278 2024-11-25T10:56:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:24,258 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:24,259 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/6b44a97a59be4940969ed28cb8141545 is 50, key is test_row_0/A:col10/1732532183645/Put/seqid=0 2024-11-25T10:56:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T10:56:24,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T10:56:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:24,297 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:56:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:24,302 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#202 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:24,303 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/5eee7d452cb342cf940a383e60af81aa is 50, key is test_row_0/B:col10/1732532183645/Put/seqid=0 2024-11-25T10:56:24,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742066_1242 (size=12663) 2024-11-25T10:56:24,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/10381fd6cc43497191d7142f7fd665b3 is 50, key is test_row_0/A:col10/1732532183764/Put/seqid=0 2024-11-25T10:56:24,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742067_1243 (size=12663) 2024-11-25T10:56:24,355 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/6b44a97a59be4940969ed28cb8141545 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6b44a97a59be4940969ed28cb8141545 2024-11-25T10:56:24,358 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/5eee7d452cb342cf940a383e60af81aa as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/5eee7d452cb342cf940a383e60af81aa 2024-11-25T10:56:24,367 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into 5eee7d452cb342cf940a383e60af81aa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:24,368 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:24,368 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532184230; duration=0sec 2024-11-25T10:56:24,368 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:24,368 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:24,368 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:24,373 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 6b44a97a59be4940969ed28cb8141545(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:24,373 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:24,373 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532184229; duration=0sec 2024-11-25T10:56:24,373 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:24,373 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:24,373 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:24,373 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:24,373 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:24,373 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/69bd51f7d89046958c0c43084b336bbf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0e850dbf4dcc40039908921db8634395, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/bd9e6b54b6db454c8b77ad6167132697] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.0 K 2024-11-25T10:56:24,374 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 69bd51f7d89046958c0c43084b336bbf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732532181270 2024-11-25T10:56:24,374 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e850dbf4dcc40039908921db8634395, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732532182468 2024-11-25T10:56:24,375 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bd9e6b54b6db454c8b77ad6167132697, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532183278 2024-11-25T10:56:24,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742068_1244 (size=12151) 2024-11-25T10:56:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:24,436 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:24,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:24,437 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/2e3a068bedcb41f48168f9a545647261 is 50, key is test_row_0/C:col10/1732532183645/Put/seqid=0 2024-11-25T10:56:24,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532244469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532244473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742069_1245 (size=12663) 2024-11-25T10:56:24,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532244477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532244584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532244584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532244597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,778 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/10381fd6cc43497191d7142f7fd665b3 2024-11-25T10:56:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T10:56:24,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/34b4ff868f004bad9816813de7d8fef4 is 50, key is test_row_0/B:col10/1732532183764/Put/seqid=0 2024-11-25T10:56:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532244799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532244799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532244817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:24,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742070_1246 (size=12151) 2024-11-25T10:56:24,868 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/34b4ff868f004bad9816813de7d8fef4 2024-11-25T10:56:24,907 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/2e3a068bedcb41f48168f9a545647261 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2e3a068bedcb41f48168f9a545647261 2024-11-25T10:56:24,917 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 2e3a068bedcb41f48168f9a545647261(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:24,917 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:24,917 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532184240; duration=0sec 2024-11-25T10:56:24,917 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:24,917 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:24,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/11fb87b2104d4ca790949877960a02cf is 50, key is test_row_0/C:col10/1732532183764/Put/seqid=0 2024-11-25T10:56:24,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742071_1247 (size=12151) 2024-11-25T10:56:25,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532245117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532245120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532245133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,369 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/11fb87b2104d4ca790949877960a02cf 2024-11-25T10:56:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/10381fd6cc43497191d7142f7fd665b3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/10381fd6cc43497191d7142f7fd665b3 2024-11-25T10:56:25,425 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/10381fd6cc43497191d7142f7fd665b3, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T10:56:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/34b4ff868f004bad9816813de7d8fef4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/34b4ff868f004bad9816813de7d8fef4 2024-11-25T10:56:25,441 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/34b4ff868f004bad9816813de7d8fef4, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T10:56:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/11fb87b2104d4ca790949877960a02cf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/11fb87b2104d4ca790949877960a02cf 2024-11-25T10:56:25,479 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/11fb87b2104d4ca790949877960a02cf, entries=150, sequenceid=245, filesize=11.9 K 2024-11-25T10:56:25,482 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d938fc35c4eb636e5192d194067fd994 in 1185ms, sequenceid=245, compaction requested=false 2024-11-25T10:56:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-25T10:56:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-25T10:56:25,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-25T10:56:25,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-11-25T10:56:25,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8720 sec 2024-11-25T10:56:25,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T10:56:25,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:25,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:25,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:25,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:25,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:25,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:25,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/9c2aea282f604a71bc89435f65b981c4 is 50, key is test_row_0/A:col10/1732532185639/Put/seqid=0 2024-11-25T10:56:25,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742072_1248 (size=12301) 2024-11-25T10:56:25,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532245706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532245711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532245713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T10:56:25,803 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:25,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-25T10:56:25,806 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:25,807 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:25,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T10:56:25,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532245823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532245823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532245834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T10:56:25,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:25,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:25,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:25,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:25,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1732532246003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,008 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_1' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8337 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36070 deadline: 1732532246014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,019 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8342 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532246033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532246033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532246044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/9c2aea282f604a71bc89435f65b981c4 2024-11-25T10:56:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T10:56:26,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/e73e43efce7d4348b7d9ba7501ea8202 is 50, key is test_row_0/B:col10/1732532185639/Put/seqid=0 2024-11-25T10:56:26,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742073_1249 (size=12301) 2024-11-25T10:56:26,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/e73e43efce7d4348b7d9ba7501ea8202 2024-11-25T10:56:26,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/19086a92101d4a04ae06af96fa88584f is 50, key is test_row_0/C:col10/1732532185639/Put/seqid=0 2024-11-25T10:56:26,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742074_1250 (size=12301) 2024-11-25T10:56:26,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/19086a92101d4a04ae06af96fa88584f 2024-11-25T10:56:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/9c2aea282f604a71bc89435f65b981c4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9c2aea282f604a71bc89435f65b981c4 2024-11-25T10:56:26,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9c2aea282f604a71bc89435f65b981c4, entries=150, sequenceid=262, filesize=12.0 K 2024-11-25T10:56:26,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:26,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:26,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/e73e43efce7d4348b7d9ba7501ea8202 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e73e43efce7d4348b7d9ba7501ea8202 2024-11-25T10:56:26,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e73e43efce7d4348b7d9ba7501ea8202, entries=150, sequenceid=262, filesize=12.0 K 2024-11-25T10:56:26,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/19086a92101d4a04ae06af96fa88584f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/19086a92101d4a04ae06af96fa88584f 2024-11-25T10:56:26,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532246343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532246343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/19086a92101d4a04ae06af96fa88584f, entries=150, sequenceid=262, filesize=12.0 K 2024-11-25T10:56:26,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d938fc35c4eb636e5192d194067fd994 in 711ms, sequenceid=262, compaction requested=true 2024-11-25T10:56:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:26,351 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:26,351 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:26,354 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:26,354 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:26,354 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,354 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6b44a97a59be4940969ed28cb8141545, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/10381fd6cc43497191d7142f7fd665b3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9c2aea282f604a71bc89435f65b981c4] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.2 K 2024-11-25T10:56:26,358 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b44a97a59be4940969ed28cb8141545, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532183278 2024-11-25T10:56:26,358 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10381fd6cc43497191d7142f7fd665b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532183716 2024-11-25T10:56:26,359 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2aea282f604a71bc89435f65b981c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732532184462 2024-11-25T10:56:26,361 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:26,361 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:26,361 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,361 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/5eee7d452cb342cf940a383e60af81aa, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/34b4ff868f004bad9816813de7d8fef4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e73e43efce7d4348b7d9ba7501ea8202] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.2 K 2024-11-25T10:56:26,362 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eee7d452cb342cf940a383e60af81aa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532183278 2024-11-25T10:56:26,363 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 34b4ff868f004bad9816813de7d8fef4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532183716 2024-11-25T10:56:26,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:56:26,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:26,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:26,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:26,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:26,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:26,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:26,366 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e73e43efce7d4348b7d9ba7501ea8202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732532184462 2024-11-25T10:56:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:26,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/364e77fe757e41688a4196afadf83bde is 50, key is test_row_0/A:col10/1732532185709/Put/seqid=0 2024-11-25T10:56:26,389 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:26,390 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1ef82d41f6c14f0e87e0721e628fa259 is 50, key is test_row_0/A:col10/1732532185639/Put/seqid=0 2024-11-25T10:56:26,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742075_1251 (size=14741) 2024-11-25T10:56:26,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/364e77fe757e41688a4196afadf83bde 2024-11-25T10:56:26,424 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:26,425 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a542e86e6d93410da8db612328a09536 is 50, key is test_row_0/B:col10/1732532185639/Put/seqid=0 2024-11-25T10:56:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T10:56:26,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:26,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:26,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/14094819984a499789977cc1fb0de247 is 50, key is test_row_0/B:col10/1732532185709/Put/seqid=0 2024-11-25T10:56:26,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742076_1252 (size=12915) 2024-11-25T10:56:26,467 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1ef82d41f6c14f0e87e0721e628fa259 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1ef82d41f6c14f0e87e0721e628fa259 2024-11-25T10:56:26,474 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 1ef82d41f6c14f0e87e0721e628fa259(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:26,474 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:26,474 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532186351; duration=0sec 2024-11-25T10:56:26,474 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:26,474 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:26,474 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:26,475 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:26,475 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:26,475 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,476 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2e3a068bedcb41f48168f9a545647261, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/11fb87b2104d4ca790949877960a02cf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/19086a92101d4a04ae06af96fa88584f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.2 K 2024-11-25T10:56:26,476 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e3a068bedcb41f48168f9a545647261, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732532183278 2024-11-25T10:56:26,476 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fb87b2104d4ca790949877960a02cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732532183716 2024-11-25T10:56:26,477 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19086a92101d4a04ae06af96fa88584f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732532184462 2024-11-25T10:56:26,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532246475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742077_1253 (size=12915) 2024-11-25T10:56:26,498 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:26,499 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/9309401314f0405681d6e3445fd36ec4 is 50, key is test_row_0/C:col10/1732532185639/Put/seqid=0 2024-11-25T10:56:26,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742078_1254 (size=12301) 2024-11-25T10:56:26,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742079_1255 (size=12915) 2024-11-25T10:56:26,554 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/9309401314f0405681d6e3445fd36ec4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/9309401314f0405681d6e3445fd36ec4 2024-11-25T10:56:26,561 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 9309401314f0405681d6e3445fd36ec4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:26,561 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:26,561 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532186351; duration=0sec 2024-11-25T10:56:26,561 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:26,561 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:26,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532246594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532246813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532246856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532246856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:26,902 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a542e86e6d93410da8db612328a09536 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a542e86e6d93410da8db612328a09536 2024-11-25T10:56:26,908 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into a542e86e6d93410da8db612328a09536(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:26,908 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:26,908 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532186351; duration=0sec 2024-11-25T10:56:26,909 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:26,909 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:26,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/14094819984a499789977cc1fb0de247 2024-11-25T10:56:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T10:56:26,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:26,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:26,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:26,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:26,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/45c2ae878bf44984895933f6842fabee is 50, key is test_row_0/C:col10/1732532185709/Put/seqid=0 2024-11-25T10:56:27,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742080_1256 (size=12301) 2024-11-25T10:56:27,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/45c2ae878bf44984895933f6842fabee 2024-11-25T10:56:27,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/364e77fe757e41688a4196afadf83bde as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/364e77fe757e41688a4196afadf83bde 2024-11-25T10:56:27,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/364e77fe757e41688a4196afadf83bde, entries=200, sequenceid=281, filesize=14.4 K 2024-11-25T10:56:27,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/14094819984a499789977cc1fb0de247 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/14094819984a499789977cc1fb0de247 2024-11-25T10:56:27,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:27,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/14094819984a499789977cc1fb0de247, entries=150, sequenceid=281, filesize=12.0 K 2024-11-25T10:56:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/45c2ae878bf44984895933f6842fabee as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/45c2ae878bf44984895933f6842fabee 2024-11-25T10:56:27,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/45c2ae878bf44984895933f6842fabee, entries=150, sequenceid=281, filesize=12.0 K 2024-11-25T10:56:27,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d938fc35c4eb636e5192d194067fd994 in 763ms, sequenceid=281, compaction requested=false 2024-11-25T10:56:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:27,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:56:27,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:27,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:27,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:27,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:27,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:27,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:27,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/3f1d64634f36429f83b1f6b907e6e4bb is 50, key is test_row_0/A:col10/1732532187132/Put/seqid=0 2024-11-25T10:56:27,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742081_1257 (size=12301) 2024-11-25T10:56:27,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532247321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:27,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:27,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:27,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532247433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:27,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/3f1d64634f36429f83b1f6b907e6e4bb 2024-11-25T10:56:27,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/d498e3a09c14406aa379775bc89f89cb is 50, key is test_row_0/B:col10/1732532187132/Put/seqid=0 2024-11-25T10:56:27,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532247660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:27,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742082_1258 (size=12301) 2024-11-25T10:56:27,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/d498e3a09c14406aa379775bc89f89cb 2024-11-25T10:56:27,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/2532bed5f5ee4a08ad00cacf6e856a19 is 50, key is test_row_0/C:col10/1732532187132/Put/seqid=0 2024-11-25T10:56:27,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:27,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:27,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:27,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742083_1259 (size=12301) 2024-11-25T10:56:27,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/2532bed5f5ee4a08ad00cacf6e856a19 2024-11-25T10:56:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/3f1d64634f36429f83b1f6b907e6e4bb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3f1d64634f36429f83b1f6b907e6e4bb 2024-11-25T10:56:27,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3f1d64634f36429f83b1f6b907e6e4bb, entries=150, sequenceid=299, filesize=12.0 K 2024-11-25T10:56:27,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/d498e3a09c14406aa379775bc89f89cb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/d498e3a09c14406aa379775bc89f89cb 2024-11-25T10:56:27,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/d498e3a09c14406aa379775bc89f89cb, entries=150, sequenceid=299, filesize=12.0 K 2024-11-25T10:56:27,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532247881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:27,887 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:27,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/2532bed5f5ee4a08ad00cacf6e856a19 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2532bed5f5ee4a08ad00cacf6e856a19 2024-11-25T10:56:27,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532247886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:27,891 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in d938fc35c4eb636e5192d194067fd994 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:27,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2532bed5f5ee4a08ad00cacf6e856a19, entries=150, sequenceid=299, filesize=12.0 K 2024-11-25T10:56:27,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d938fc35c4eb636e5192d194067fd994 in 764ms, sequenceid=299, compaction requested=true 2024-11-25T10:56:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:27,905 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:27,905 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:27,909 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:27,909 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:27,909 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,909 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1ef82d41f6c14f0e87e0721e628fa259, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/364e77fe757e41688a4196afadf83bde, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3f1d64634f36429f83b1f6b907e6e4bb] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=39.0 K 2024-11-25T10:56:27,909 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:27,909 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:27,910 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,910 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a542e86e6d93410da8db612328a09536, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/14094819984a499789977cc1fb0de247, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/d498e3a09c14406aa379775bc89f89cb] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.6 K 2024-11-25T10:56:27,910 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef82d41f6c14f0e87e0721e628fa259, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732532184462 2024-11-25T10:56:27,910 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a542e86e6d93410da8db612328a09536, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732532184462 2024-11-25T10:56:27,910 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 14094819984a499789977cc1fb0de247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732532185660 2024-11-25T10:56:27,911 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d498e3a09c14406aa379775bc89f89cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732532186458 2024-11-25T10:56:27,912 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 364e77fe757e41688a4196afadf83bde, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732532185660 2024-11-25T10:56:27,913 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f1d64634f36429f83b1f6b907e6e4bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732532186458 2024-11-25T10:56:27,922 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:27,926 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/560cfbfbd58743e38208ded2ea49900d is 50, key is test_row_0/B:col10/1732532187132/Put/seqid=0 2024-11-25T10:56:27,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T10:56:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:27,941 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:56:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T10:56:27,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742084_1260 (size=13017) 2024-11-25T10:56:27,985 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:27,986 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/6e5ccb6c6c284ce58027fe76422387d4 is 50, key is test_row_0/A:col10/1732532187132/Put/seqid=0 2024-11-25T10:56:27,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/4bdfaa51743a47bd9fa875f1ace94cb8 is 50, key is test_row_0/A:col10/1732532187317/Put/seqid=0 2024-11-25T10:56:27,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:28,010 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/560cfbfbd58743e38208ded2ea49900d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/560cfbfbd58743e38208ded2ea49900d 2024-11-25T10:56:28,024 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into 560cfbfbd58743e38208ded2ea49900d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:28,024 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:28,024 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532187905; duration=0sec 2024-11-25T10:56:28,025 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:28,025 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:28,025 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:28,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742085_1261 (size=13017) 2024-11-25T10:56:28,030 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:28,031 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:28,031 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:28,031 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/9309401314f0405681d6e3445fd36ec4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/45c2ae878bf44984895933f6842fabee, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2532bed5f5ee4a08ad00cacf6e856a19] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.6 K 2024-11-25T10:56:28,031 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9309401314f0405681d6e3445fd36ec4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732532184462 2024-11-25T10:56:28,032 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c2ae878bf44984895933f6842fabee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732532185660 2024-11-25T10:56:28,033 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2532bed5f5ee4a08ad00cacf6e856a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732532186458 2024-11-25T10:56:28,047 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/6e5ccb6c6c284ce58027fe76422387d4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6e5ccb6c6c284ce58027fe76422387d4 2024-11-25T10:56:28,061 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:28,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742086_1262 (size=12301) 2024-11-25T10:56:28,062 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/2d7cbc7d82ff4898aa0e250e933028f8 is 50, key is test_row_0/C:col10/1732532187132/Put/seqid=0 2024-11-25T10:56:28,071 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/4bdfaa51743a47bd9fa875f1ace94cb8 2024-11-25T10:56:28,089 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 6e5ccb6c6c284ce58027fe76422387d4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:28,089 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:28,089 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532187905; duration=0sec 2024-11-25T10:56:28,090 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:28,090 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:28,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0c88c518c99a4d69ab1b15501c566533 is 50, key is test_row_0/B:col10/1732532187317/Put/seqid=0 2024-11-25T10:56:28,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742087_1263 (size=13017) 2024-11-25T10:56:28,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742088_1264 (size=12301) 2024-11-25T10:56:28,224 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0c88c518c99a4d69ab1b15501c566533 2024-11-25T10:56:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532248230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0628a4a621084c04a9f6f963b47df2e7 is 50, key is test_row_0/C:col10/1732532187317/Put/seqid=0 2024-11-25T10:56:28,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742089_1265 (size=12301) 2024-11-25T10:56:28,337 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0628a4a621084c04a9f6f963b47df2e7 2024-11-25T10:56:28,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532248357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:28,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/4bdfaa51743a47bd9fa875f1ace94cb8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/4bdfaa51743a47bd9fa875f1ace94cb8 2024-11-25T10:56:28,422 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/4bdfaa51743a47bd9fa875f1ace94cb8, entries=150, sequenceid=316, filesize=12.0 K 2024-11-25T10:56:28,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0c88c518c99a4d69ab1b15501c566533 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c88c518c99a4d69ab1b15501c566533 2024-11-25T10:56:28,451 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c88c518c99a4d69ab1b15501c566533, entries=150, sequenceid=316, filesize=12.0 K 2024-11-25T10:56:28,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0628a4a621084c04a9f6f963b47df2e7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0628a4a621084c04a9f6f963b47df2e7 2024-11-25T10:56:28,490 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0628a4a621084c04a9f6f963b47df2e7, entries=150, sequenceid=316, filesize=12.0 K 2024-11-25T10:56:28,492 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d938fc35c4eb636e5192d194067fd994 in 551ms, sequenceid=316, compaction requested=false 2024-11-25T10:56:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-25T10:56:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-25T10:56:28,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-25T10:56:28,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6870 sec 2024-11-25T10:56:28,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.6950 sec 2024-11-25T10:56:28,568 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/2d7cbc7d82ff4898aa0e250e933028f8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2d7cbc7d82ff4898aa0e250e933028f8 2024-11-25T10:56:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:28,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T10:56:28,592 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 2d7cbc7d82ff4898aa0e250e933028f8(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:28,593 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:28,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:28,593 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532187905; duration=0sec 2024-11-25T10:56:28,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:28,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:28,593 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:28,593 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:28,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:28,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:28,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:28,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/488bd1cee4994318aed203ec83823ee8 is 50, key is test_row_0/A:col10/1732532188058/Put/seqid=0 2024-11-25T10:56:28,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742090_1266 (size=14741) 2024-11-25T10:56:28,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532248790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:28,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532248917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:29,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/488bd1cee4994318aed203ec83823ee8 2024-11-25T10:56:29,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0cd7273cf3ee4c5685da371403dc1abf is 50, key is test_row_0/B:col10/1732532188058/Put/seqid=0 2024-11-25T10:56:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532249141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:29,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742091_1267 (size=12301) 2024-11-25T10:56:29,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532249457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:29,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0cd7273cf3ee4c5685da371403dc1abf 2024-11-25T10:56:29,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/a66132d290744387b26826501cd677df is 50, key is test_row_0/C:col10/1732532188058/Put/seqid=0 2024-11-25T10:56:29,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742092_1268 (size=12301) 2024-11-25T10:56:29,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/a66132d290744387b26826501cd677df 2024-11-25T10:56:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/488bd1cee4994318aed203ec83823ee8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/488bd1cee4994318aed203ec83823ee8 2024-11-25T10:56:29,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/488bd1cee4994318aed203ec83823ee8, entries=200, sequenceid=336, filesize=14.4 K 2024-11-25T10:56:29,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0cd7273cf3ee4c5685da371403dc1abf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0cd7273cf3ee4c5685da371403dc1abf 2024-11-25T10:56:29,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0cd7273cf3ee4c5685da371403dc1abf, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T10:56:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/a66132d290744387b26826501cd677df as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/a66132d290744387b26826501cd677df 2024-11-25T10:56:29,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/a66132d290744387b26826501cd677df, entries=150, sequenceid=336, filesize=12.0 K 2024-11-25T10:56:29,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d938fc35c4eb636e5192d194067fd994 in 1202ms, sequenceid=336, compaction requested=true 2024-11-25T10:56:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:29,781 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:29,781 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:29,782 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:29,782 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:29,782 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:29,783 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6e5ccb6c6c284ce58027fe76422387d4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/4bdfaa51743a47bd9fa875f1ace94cb8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/488bd1cee4994318aed203ec83823ee8] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=39.1 K 2024-11-25T10:56:29,783 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5ccb6c6c284ce58027fe76422387d4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732532186458 2024-11-25T10:56:29,783 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:29,783 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:29,783 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bdfaa51743a47bd9fa875f1ace94cb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732532187252 2024-11-25T10:56:29,783 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:29,784 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/560cfbfbd58743e38208ded2ea49900d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c88c518c99a4d69ab1b15501c566533, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0cd7273cf3ee4c5685da371403dc1abf] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.7 K 2024-11-25T10:56:29,784 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 488bd1cee4994318aed203ec83823ee8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732532188058 2024-11-25T10:56:29,784 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 560cfbfbd58743e38208ded2ea49900d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732532186458 2024-11-25T10:56:29,785 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c88c518c99a4d69ab1b15501c566533, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732532187252 2024-11-25T10:56:29,785 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd7273cf3ee4c5685da371403dc1abf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732532188058 2024-11-25T10:56:29,797 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:29,798 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/66b22c6129fe44c3a063f28c90b0232e is 50, key is test_row_0/B:col10/1732532188058/Put/seqid=0 2024-11-25T10:56:29,810 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:29,810 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0498ae2e24bb475eaefdc3b4fcce2e95 is 50, key is test_row_0/A:col10/1732532188058/Put/seqid=0 2024-11-25T10:56:29,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742094_1270 (size=13119) 2024-11-25T10:56:29,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742093_1269 (size=13119) 2024-11-25T10:56:29,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T10:56:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:29,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/3d06d5f393414699ae5db77c1c865097 is 50, key is test_row_0/A:col10/1732532188702/Put/seqid=0 2024-11-25T10:56:29,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742095_1271 (size=14741) 2024-11-25T10:56:29,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/3d06d5f393414699ae5db77c1c865097 2024-11-25T10:56:29,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0c75ac853e664110bd9b13e514372107 is 50, key is test_row_0/B:col10/1732532188702/Put/seqid=0 2024-11-25T10:56:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T10:56:29,992 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:29,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:29,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532249983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:29,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532249989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-25T10:56:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T10:56:29,998 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:29,999 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:29,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532249992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742096_1272 (size=12301) 2024-11-25T10:56:30,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0c75ac853e664110bd9b13e514372107 2024-11-25T10:56:30,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/e08b343c93484630bd2c5c49f9245bcf is 50, key is test_row_0/C:col10/1732532188702/Put/seqid=0 2024-11-25T10:56:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T10:56:30,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532250104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532250104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742097_1273 (size=12301) 2024-11-25T10:56:30,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532250113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/e08b343c93484630bd2c5c49f9245bcf 2024-11-25T10:56:30,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T10:56:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:30,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:30,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/3d06d5f393414699ae5db77c1c865097 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3d06d5f393414699ae5db77c1c865097 2024-11-25T10:56:30,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3d06d5f393414699ae5db77c1c865097, entries=200, sequenceid=352, filesize=14.4 K 2024-11-25T10:56:30,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/0c75ac853e664110bd9b13e514372107 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c75ac853e664110bd9b13e514372107 2024-11-25T10:56:30,309 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0498ae2e24bb475eaefdc3b4fcce2e95 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0498ae2e24bb475eaefdc3b4fcce2e95 2024-11-25T10:56:30,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c75ac853e664110bd9b13e514372107, entries=150, sequenceid=352, filesize=12.0 K 2024-11-25T10:56:30,310 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/66b22c6129fe44c3a063f28c90b0232e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/66b22c6129fe44c3a063f28c90b0232e 2024-11-25T10:56:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T10:56:30,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/e08b343c93484630bd2c5c49f9245bcf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e08b343c93484630bd2c5c49f9245bcf 2024-11-25T10:56:30,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T10:56:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:30,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:30,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532250315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532250315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532250330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,360 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 0498ae2e24bb475eaefdc3b4fcce2e95(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:30,360 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:30,360 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532189781; duration=0sec 2024-11-25T10:56:30,360 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:30,360 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:30,360 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:30,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e08b343c93484630bd2c5c49f9245bcf, entries=150, sequenceid=352, filesize=12.0 K 2024-11-25T10:56:30,380 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into 66b22c6129fe44c3a063f28c90b0232e(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:30,380 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:30,380 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532189781; duration=0sec 2024-11-25T10:56:30,384 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:30,385 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:30,385 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:30,385 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2d7cbc7d82ff4898aa0e250e933028f8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0628a4a621084c04a9f6f963b47df2e7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/a66132d290744387b26826501cd677df] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=36.7 K 2024-11-25T10:56:30,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d938fc35c4eb636e5192d194067fd994 in 476ms, sequenceid=352, compaction requested=false 2024-11-25T10:56:30,386 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:30,386 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:30,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:30,388 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d7cbc7d82ff4898aa0e250e933028f8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732532186458 2024-11-25T10:56:30,388 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0628a4a621084c04a9f6f963b47df2e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732532187252 2024-11-25T10:56:30,393 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66132d290744387b26826501cd677df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732532188058 2024-11-25T10:56:30,436 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:30,436 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/fbade3a9086d4b19a7ee5b8616591a4c is 50, key is test_row_0/C:col10/1732532188058/Put/seqid=0 2024-11-25T10:56:30,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T10:56:30,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:30,481 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T10:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:30,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742098_1274 (size=13119) 2024-11-25T10:56:30,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/e1e1aae829f8475a8e603a22cadc004d is 50, key is test_row_0/A:col10/1732532189952/Put/seqid=0 2024-11-25T10:56:30,556 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/fbade3a9086d4b19a7ee5b8616591a4c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/fbade3a9086d4b19a7ee5b8616591a4c 2024-11-25T10:56:30,579 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into fbade3a9086d4b19a7ee5b8616591a4c(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:30,579 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:30,579 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532189781; duration=0sec 2024-11-25T10:56:30,579 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:30,579 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:30,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742099_1275 (size=12301) 2024-11-25T10:56:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T10:56:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:30,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:30,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532250698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532250698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532250705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532250814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532250815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532250817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:30,985 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/e1e1aae829f8475a8e603a22cadc004d 2024-11-25T10:56:31,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a22091b19e3444b1b2757318d3945c78 is 50, key is test_row_0/B:col10/1732532189952/Put/seqid=0 2024-11-25T10:56:31,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532251024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532251025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532251025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742100_1276 (size=12301) 2024-11-25T10:56:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T10:56:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532251334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532251334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532251343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,461 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a22091b19e3444b1b2757318d3945c78 2024-11-25T10:56:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/be8f0d83be0d4d6582095e3ae171c7c3 is 50, key is test_row_0/C:col10/1732532189952/Put/seqid=0 2024-11-25T10:56:31,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742101_1277 (size=12301) 2024-11-25T10:56:31,548 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/be8f0d83be0d4d6582095e3ae171c7c3 2024-11-25T10:56:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/e1e1aae829f8475a8e603a22cadc004d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e1e1aae829f8475a8e603a22cadc004d 2024-11-25T10:56:31,621 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e1e1aae829f8475a8e603a22cadc004d, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T10:56:31,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/a22091b19e3444b1b2757318d3945c78 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a22091b19e3444b1b2757318d3945c78 2024-11-25T10:56:31,663 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a22091b19e3444b1b2757318d3945c78, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T10:56:31,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/be8f0d83be0d4d6582095e3ae171c7c3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/be8f0d83be0d4d6582095e3ae171c7c3 2024-11-25T10:56:31,711 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/be8f0d83be0d4d6582095e3ae171c7c3, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T10:56:31,712 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d938fc35c4eb636e5192d194067fd994 in 1232ms, sequenceid=370, compaction requested=true 2024-11-25T10:56:31,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:31,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:31,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-25T10:56:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-25T10:56:31,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-25T10:56:31,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-11-25T10:56:31,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7310 sec 2024-11-25T10:56:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:31,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:56:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:31,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:31,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/9a3f14067d314dd3aa82d24ee92c9f73 is 50, key is test_row_0/A:col10/1732532191846/Put/seqid=0 2024-11-25T10:56:31,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742102_1278 (size=12301) 2024-11-25T10:56:31,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/9a3f14067d314dd3aa82d24ee92c9f73 2024-11-25T10:56:31,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532251902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532251906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532251908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:31,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/15f21f6f0ff74b50a07de7e3d2af81fe is 50, key is test_row_0/B:col10/1732532191846/Put/seqid=0 2024-11-25T10:56:31,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742103_1279 (size=12301) 2024-11-25T10:56:32,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532252016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532252025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532252025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T10:56:32,160 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:32,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-25T10:56:32,169 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:32,169 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:32,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T10:56:32,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532252228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532252234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532252245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T10:56:32,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/15f21f6f0ff74b50a07de7e3d2af81fe 2024-11-25T10:56:32,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/08ce44a46e7e49d2b32c5621adc35f6e is 50, key is test_row_0/C:col10/1732532191846/Put/seqid=0 2024-11-25T10:56:32,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742104_1280 (size=12301) 2024-11-25T10:56:32,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/08ce44a46e7e49d2b32c5621adc35f6e 2024-11-25T10:56:32,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/9a3f14067d314dd3aa82d24ee92c9f73 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9a3f14067d314dd3aa82d24ee92c9f73 2024-11-25T10:56:32,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9a3f14067d314dd3aa82d24ee92c9f73, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T10:56:32,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/15f21f6f0ff74b50a07de7e3d2af81fe as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/15f21f6f0ff74b50a07de7e3d2af81fe 2024-11-25T10:56:32,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/15f21f6f0ff74b50a07de7e3d2af81fe, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T10:56:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T10:56:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/08ce44a46e7e49d2b32c5621adc35f6e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/08ce44a46e7e49d2b32c5621adc35f6e 2024-11-25T10:56:32,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/08ce44a46e7e49d2b32c5621adc35f6e, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T10:56:32,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d938fc35c4eb636e5192d194067fd994 in 662ms, sequenceid=389, compaction requested=true 2024-11-25T10:56:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:32,515 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:32,515 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:32,529 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:32,529 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:32,529 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,529 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/66b22c6129fe44c3a063f28c90b0232e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c75ac853e664110bd9b13e514372107, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a22091b19e3444b1b2757318d3945c78, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/15f21f6f0ff74b50a07de7e3d2af81fe] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=48.8 K 2024-11-25T10:56:32,529 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:32,529 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:32,529 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,529 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0498ae2e24bb475eaefdc3b4fcce2e95, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3d06d5f393414699ae5db77c1c865097, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e1e1aae829f8475a8e603a22cadc004d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9a3f14067d314dd3aa82d24ee92c9f73] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=51.2 K 2024-11-25T10:56:32,533 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0498ae2e24bb475eaefdc3b4fcce2e95, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732532188058 2024-11-25T10:56:32,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b22c6129fe44c3a063f28c90b0232e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732532188058 2024-11-25T10:56:32,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c75ac853e664110bd9b13e514372107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732532188702 2024-11-25T10:56:32,533 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d06d5f393414699ae5db77c1c865097, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732532188702 2024-11-25T10:56:32,541 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e1aae829f8475a8e603a22cadc004d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732532189952 2024-11-25T10:56:32,541 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a22091b19e3444b1b2757318d3945c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732532189952 2024-11-25T10:56:32,545 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a3f14067d314dd3aa82d24ee92c9f73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732532190678 2024-11-25T10:56:32,545 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 15f21f6f0ff74b50a07de7e3d2af81fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732532190678 2024-11-25T10:56:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:32,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:56:32,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:32,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:32,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:32,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:32,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:32,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:32,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/e160cc68dfce4f9caed5c318deaab068 is 50, key is test_row_0/A:col10/1732532191874/Put/seqid=0 2024-11-25T10:56:32,608 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:32,608 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0dc2953ec8b149da8ca623423701f889 is 50, key is test_row_0/A:col10/1732532191846/Put/seqid=0 2024-11-25T10:56:32,612 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:32,613 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/9e77e4467eae4e5082d55d40425a7f60 is 50, key is test_row_0/B:col10/1732532191846/Put/seqid=0 2024-11-25T10:56:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532252622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532252633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532252634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742105_1281 (size=14741) 2024-11-25T10:56:32,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:32,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742106_1282 (size=13255) 2024-11-25T10:56:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742107_1283 (size=13255) 2024-11-25T10:56:32,744 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/0dc2953ec8b149da8ca623423701f889 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0dc2953ec8b149da8ca623423701f889 2024-11-25T10:56:32,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532252744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532252754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,763 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into 0dc2953ec8b149da8ca623423701f889(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:32,763 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:32,763 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=12, startTime=1732532192515; duration=0sec 2024-11-25T10:56:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,763 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,763 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,763 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:32,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532252757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,783 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:32,783 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:32,783 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,783 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/fbade3a9086d4b19a7ee5b8616591a4c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e08b343c93484630bd2c5c49f9245bcf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/be8f0d83be0d4d6582095e3ae171c7c3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/08ce44a46e7e49d2b32c5621adc35f6e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=48.8 K 2024-11-25T10:56:32,794 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbade3a9086d4b19a7ee5b8616591a4c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732532188058 2024-11-25T10:56:32,800 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08b343c93484630bd2c5c49f9245bcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732532188702 2024-11-25T10:56:32,801 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting be8f0d83be0d4d6582095e3ae171c7c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732532189952 2024-11-25T10:56:32,808 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ce44a46e7e49d2b32c5621adc35f6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732532190678 2024-11-25T10:56:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T10:56:32,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,881 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:32,882 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0eeca873a5334e1cb75c348e301b422f is 50, key is test_row_0/C:col10/1732532191846/Put/seqid=0 2024-11-25T10:56:32,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742108_1284 (size=13255) 2024-11-25T10:56:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532252960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532252976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532252976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:32,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:32,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/e160cc68dfce4f9caed5c318deaab068 2024-11-25T10:56:33,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/2710204ab91f4a579e5a08c32e3b2fb6 is 50, key is test_row_0/B:col10/1732532191874/Put/seqid=0 2024-11-25T10:56:33,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742109_1285 (size=12301) 2024-11-25T10:56:33,138 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/9e77e4467eae4e5082d55d40425a7f60 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9e77e4467eae4e5082d55d40425a7f60 2024-11-25T10:56:33,153 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into 9e77e4467eae4e5082d55d40425a7f60(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:33,153 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:33,153 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=12, startTime=1732532192515; duration=0sec 2024-11-25T10:56:33,153 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:33,153 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:33,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:33,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36044 deadline: 1732532253274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:33,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36002 deadline: 1732532253293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:33,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36080 deadline: 1732532253293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:33,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:56:33,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:33,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T10:56:33,387 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/0eeca873a5334e1cb75c348e301b422f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0eeca873a5334e1cb75c348e301b422f 2024-11-25T10:56:33,429 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 0eeca873a5334e1cb75c348e301b422f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:33,429 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:33,429 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=12, startTime=1732532192515; duration=0sec 2024-11-25T10:56:33,429 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:33,429 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:33,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/2710204ab91f4a579e5a08c32e3b2fb6 2024-11-25T10:56:33,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/ba940297334a484096e5686c16ead451 is 50, key is test_row_0/C:col10/1732532191874/Put/seqid=0 2024-11-25T10:56:33,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742110_1286 (size=12301) 2024-11-25T10:56:33,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/ba940297334a484096e5686c16ead451 2024-11-25T10:56:33,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/e160cc68dfce4f9caed5c318deaab068 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e160cc68dfce4f9caed5c318deaab068 2024-11-25T10:56:33,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e160cc68dfce4f9caed5c318deaab068, entries=200, sequenceid=404, filesize=14.4 K 2024-11-25T10:56:33,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/2710204ab91f4a579e5a08c32e3b2fb6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2710204ab91f4a579e5a08c32e3b2fb6 2024-11-25T10:56:33,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:33,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:33,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2710204ab91f4a579e5a08c32e3b2fb6, entries=150, sequenceid=404, filesize=12.0 K 2024-11-25T10:56:33,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/ba940297334a484096e5686c16ead451 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ba940297334a484096e5686c16ead451 2024-11-25T10:56:33,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ba940297334a484096e5686c16ead451, entries=150, sequenceid=404, filesize=12.0 K 2024-11-25T10:56:33,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for d938fc35c4eb636e5192d194067fd994 in 1157ms, sequenceid=404, compaction requested=false 2024-11-25T10:56:33,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:33,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:56:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:33,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:33,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:33,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:33,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:33,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:33,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:33,807 INFO [Thread-944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-944. 2024-11-25T10:56:33,807 DEBUG [Thread-944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,807 DEBUG [Thread-944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,808 DEBUG [Thread-944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:33,809 INFO [Thread-957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-957. 2024-11-25T10:56:33,809 DEBUG [Thread-957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,809 DEBUG [Thread-957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:33,809 DEBUG [Thread-957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,814 INFO [Thread-950 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-950. 2024-11-25T10:56:33,814 DEBUG [Thread-950 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,814 DEBUG [Thread-950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,814 DEBUG [Thread-950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,817 INFO [Thread-955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-955. 2024-11-25T10:56:33,817 DEBUG [Thread-955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,817 DEBUG [Thread-955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,817 DEBUG [Thread-955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,818 INFO [Thread-959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-959. 2024-11-25T10:56:33,818 DEBUG [Thread-959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,818 DEBUG [Thread-959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,818 DEBUG [Thread-959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,820 INFO [Thread-961 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-961. 2024-11-25T10:56:33,820 DEBUG [Thread-961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,821 DEBUG [Thread-961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,821 DEBUG [Thread-961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,821 INFO [Thread-953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-953. 2024-11-25T10:56:33,821 DEBUG [Thread-953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,821 DEBUG [Thread-953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,821 DEBUG [Thread-953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/670aa7c06bda4891adceada8ccca4a07 is 50, key is test_row_0/A:col10/1732532192598/Put/seqid=0 2024-11-25T10:56:33,828 INFO [Thread-948 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-948. 2024-11-25T10:56:33,828 DEBUG [Thread-948 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:33,828 DEBUG [Thread-948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,829 DEBUG [Thread-948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:33,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:33,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742111_1287 (size=12301) 2024-11-25T10:56:33,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:33,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:33,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:33,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:34,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:34,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:34,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:34,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:34,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/670aa7c06bda4891adceada8ccca4a07 2024-11-25T10:56:34,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/1c8e53b6e1dc49bbaab812ad959e1c5e is 50, key is test_row_0/B:col10/1732532192598/Put/seqid=0 2024-11-25T10:56:34,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:34,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. as already flushing 2024-11-25T10:56:34,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:34,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:34,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742112_1288 (size=12301) 2024-11-25T10:56:34,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/1c8e53b6e1dc49bbaab812ad959e1c5e 2024-11-25T10:56:34,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/13b0ee57074b4ba691427f02f7da6014 is 50, key is test_row_0/C:col10/1732532192598/Put/seqid=0 2024-11-25T10:56:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T10:56:34,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742113_1289 (size=12301) 2024-11-25T10:56:34,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/13b0ee57074b4ba691427f02f7da6014 2024-11-25T10:56:34,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/670aa7c06bda4891adceada8ccca4a07 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/670aa7c06bda4891adceada8ccca4a07 2024-11-25T10:56:34,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/670aa7c06bda4891adceada8ccca4a07, entries=150, sequenceid=426, filesize=12.0 K 2024-11-25T10:56:34,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/1c8e53b6e1dc49bbaab812ad959e1c5e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1c8e53b6e1dc49bbaab812ad959e1c5e 2024-11-25T10:56:34,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1c8e53b6e1dc49bbaab812ad959e1c5e, entries=150, sequenceid=426, filesize=12.0 K 2024-11-25T10:56:34,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/13b0ee57074b4ba691427f02f7da6014 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/13b0ee57074b4ba691427f02f7da6014 2024-11-25T10:56:34,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/13b0ee57074b4ba691427f02f7da6014, entries=150, sequenceid=426, filesize=12.0 K 2024-11-25T10:56:34,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for d938fc35c4eb636e5192d194067fd994 in 619ms, sequenceid=426, compaction requested=true 2024-11-25T10:56:34,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:34,408 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d938fc35c4eb636e5192d194067fd994:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:34,410 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:34,410 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/A is initiating minor compaction (all files) 2024-11-25T10:56:34,410 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/A in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,410 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0dc2953ec8b149da8ca623423701f889, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e160cc68dfce4f9caed5c318deaab068, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/670aa7c06bda4891adceada8ccca4a07] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=39.4 K 2024-11-25T10:56:34,411 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc2953ec8b149da8ca623423701f889, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732532190678 2024-11-25T10:56:34,411 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e160cc68dfce4f9caed5c318deaab068, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732532191874 2024-11-25T10:56:34,412 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:34,413 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 670aa7c06bda4891adceada8ccca4a07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732532192598 2024-11-25T10:56:34,417 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:34,417 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/B is initiating minor compaction (all files) 2024-11-25T10:56:34,418 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/B in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,418 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9e77e4467eae4e5082d55d40425a7f60, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2710204ab91f4a579e5a08c32e3b2fb6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1c8e53b6e1dc49bbaab812ad959e1c5e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=37.0 K 2024-11-25T10:56:34,418 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e77e4467eae4e5082d55d40425a7f60, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732532190678 2024-11-25T10:56:34,418 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2710204ab91f4a579e5a08c32e3b2fb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732532191874 2024-11-25T10:56:34,419 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c8e53b6e1dc49bbaab812ad959e1c5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732532192598 2024-11-25T10:56:34,428 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:34,429 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/b1620146841d47a282119e94b50eb34d is 50, key is test_row_0/A:col10/1732532192598/Put/seqid=0 2024-11-25T10:56:34,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T10:56:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,437 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-25T10:56:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:34,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:34,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:34,459 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:34,460 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/1ad67637ed8d4b1dbd76d79ce9cb08fd is 50, key is test_row_0/B:col10/1732532192598/Put/seqid=0 2024-11-25T10:56:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/7d5f39d1fdef434596fa77c337c5d152 is 50, key is test_row_0/A:col10/1732532193813/Put/seqid=0 2024-11-25T10:56:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742115_1291 (size=13357) 2024-11-25T10:56:34,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742114_1290 (size=13357) 2024-11-25T10:56:34,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742116_1292 (size=12301) 2024-11-25T10:56:34,541 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/7d5f39d1fdef434596fa77c337c5d152 2024-11-25T10:56:34,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/86700884855a42d7b436d032d0425fbc is 50, key is test_row_0/B:col10/1732532193813/Put/seqid=0 2024-11-25T10:56:34,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742117_1293 (size=12301) 2024-11-25T10:56:34,925 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/1ad67637ed8d4b1dbd76d79ce9cb08fd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1ad67637ed8d4b1dbd76d79ce9cb08fd 2024-11-25T10:56:34,927 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/b1620146841d47a282119e94b50eb34d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b1620146841d47a282119e94b50eb34d 2024-11-25T10:56:34,945 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/B of d938fc35c4eb636e5192d194067fd994 into 1ad67637ed8d4b1dbd76d79ce9cb08fd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:34,945 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:34,945 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/B, priority=13, startTime=1732532194409; duration=0sec 2024-11-25T10:56:34,945 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:34,945 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:B 2024-11-25T10:56:34,945 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:34,946 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:34,946 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): d938fc35c4eb636e5192d194067fd994/C is initiating minor compaction (all files) 2024-11-25T10:56:34,946 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d938fc35c4eb636e5192d194067fd994/C in TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:34,946 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0eeca873a5334e1cb75c348e301b422f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ba940297334a484096e5686c16ead451, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/13b0ee57074b4ba691427f02f7da6014] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp, totalSize=37.0 K 2024-11-25T10:56:34,947 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eeca873a5334e1cb75c348e301b422f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732532190678 2024-11-25T10:56:34,947 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/A of d938fc35c4eb636e5192d194067fd994 into b1620146841d47a282119e94b50eb34d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:34,947 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:34,947 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/A, priority=13, startTime=1732532194408; duration=0sec 2024-11-25T10:56:34,947 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:34,947 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:A 2024-11-25T10:56:34,948 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba940297334a484096e5686c16ead451, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732532191874 2024-11-25T10:56:34,948 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b0ee57074b4ba691427f02f7da6014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732532192598 2024-11-25T10:56:34,956 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d938fc35c4eb636e5192d194067fd994#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:34,956 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/8d35ce9ea8dc4f4e983e3695bc842f0d is 50, key is test_row_0/C:col10/1732532192598/Put/seqid=0 2024-11-25T10:56:34,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742118_1294 (size=13357) 2024-11-25T10:56:35,014 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/86700884855a42d7b436d032d0425fbc 2024-11-25T10:56:35,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/f95370670086406cb10d4acb89958c9c is 50, key is test_row_0/C:col10/1732532193813/Put/seqid=0 2024-11-25T10:56:35,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742119_1295 (size=12301) 2024-11-25T10:56:35,371 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/8d35ce9ea8dc4f4e983e3695bc842f0d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/8d35ce9ea8dc4f4e983e3695bc842f0d 2024-11-25T10:56:35,378 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d938fc35c4eb636e5192d194067fd994/C of d938fc35c4eb636e5192d194067fd994 into 8d35ce9ea8dc4f4e983e3695bc842f0d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:35,378 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:35,378 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., storeName=d938fc35c4eb636e5192d194067fd994/C, priority=13, startTime=1732532194409; duration=0sec 2024-11-25T10:56:35,378 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:35,378 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d938fc35c4eb636e5192d194067fd994:C 2024-11-25T10:56:35,426 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/f95370670086406cb10d4acb89958c9c 2024-11-25T10:56:35,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/7d5f39d1fdef434596fa77c337c5d152 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7d5f39d1fdef434596fa77c337c5d152 2024-11-25T10:56:35,438 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7d5f39d1fdef434596fa77c337c5d152, entries=150, sequenceid=433, filesize=12.0 K 2024-11-25T10:56:35,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/86700884855a42d7b436d032d0425fbc as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/86700884855a42d7b436d032d0425fbc 2024-11-25T10:56:35,448 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/86700884855a42d7b436d032d0425fbc, entries=150, sequenceid=433, filesize=12.0 K 2024-11-25T10:56:35,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/f95370670086406cb10d4acb89958c9c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f95370670086406cb10d4acb89958c9c 2024-11-25T10:56:35,453 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f95370670086406cb10d4acb89958c9c, entries=150, sequenceid=433, filesize=12.0 K 2024-11-25T10:56:35,460 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d938fc35c4eb636e5192d194067fd994 in 1023ms, sequenceid=433, compaction requested=false 2024-11-25T10:56:35,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d938fc35c4eb636e5192d194067fd994: 2024-11-25T10:56:35,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:35,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-25T10:56:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-25T10:56:35,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-25T10:56:35,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3070 sec 2024-11-25T10:56:35,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 3.3410 sec 2024-11-25T10:56:36,066 INFO [Thread-942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-942. 2024-11-25T10:56:36,066 DEBUG [Thread-942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:36,066 DEBUG [Thread-942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:36,067 DEBUG [Thread-942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:36,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:36,086 INFO [Thread-946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-946. 2024-11-25T10:56:36,086 DEBUG [Thread-946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:56:36,087 DEBUG [Thread-946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:36,087 DEBUG [Thread-946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:36,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T10:56:36,369 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:36,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T10:56:36,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-25T10:56:36,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-25T10:56:36,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-25T10:56:36,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-25T10:56:36,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-25T10:56:36,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T10:56:36,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1537 2024-11-25T10:56:36,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1516 2024-11-25T10:56:36,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1606 2024-11-25T10:56:36,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1543 2024-11-25T10:56:36,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1526 2024-11-25T10:56:36,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T10:56:36,373 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:56:36,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:56:36,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:36,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:36,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:56:36,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T10:56:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:36,392 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532196391"}]},"ts":"1732532196391"} 2024-11-25T10:56:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T10:56:36,398 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T10:56:36,398 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T10:56:36,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:56:36,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, UNASSIGN}] 2024-11-25T10:56:36,416 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, UNASSIGN 2024-11-25T10:56:36,421 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=d938fc35c4eb636e5192d194067fd994, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:36,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, UNASSIGN because future has completed 2024-11-25T10:56:36,429 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:56:36,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:56:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T10:56:36,604 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing d938fc35c4eb636e5192d194067fd994, disabling compactions & flushes 2024-11-25T10:56:36,605 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. after waiting 0 ms 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:36,605 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing d938fc35c4eb636e5192d194067fd994 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=A 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:36,605 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=B 2024-11-25T10:56:36,606 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:36,606 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d938fc35c4eb636e5192d194067fd994, store=C 2024-11-25T10:56:36,606 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:36,629 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1e9fec1845c74f17a42e610f442b6ec3 is 50, key is test_row_0/A:col10/1732532196084/Put/seqid=0 2024-11-25T10:56:36,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742120_1296 (size=9857) 2024-11-25T10:56:36,688 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1e9fec1845c74f17a42e610f442b6ec3 2024-11-25T10:56:36,705 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/62fbd8f7f9844ba8b763ba55ac45910b is 50, key is test_row_0/B:col10/1732532196084/Put/seqid=0 2024-11-25T10:56:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742121_1297 (size=9857) 2024-11-25T10:56:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T10:56:36,726 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/62fbd8f7f9844ba8b763ba55ac45910b 2024-11-25T10:56:36,765 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/38a161f3cf114a10a03a7197c374e850 is 50, key is test_row_0/C:col10/1732532196084/Put/seqid=0 2024-11-25T10:56:36,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742122_1298 (size=9857) 2024-11-25T10:56:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T10:56:37,214 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/38a161f3cf114a10a03a7197c374e850 2024-11-25T10:56:37,240 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/A/1e9fec1845c74f17a42e610f442b6ec3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1e9fec1845c74f17a42e610f442b6ec3 2024-11-25T10:56:37,297 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1e9fec1845c74f17a42e610f442b6ec3, entries=100, sequenceid=441, filesize=9.6 K 2024-11-25T10:56:37,316 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/B/62fbd8f7f9844ba8b763ba55ac45910b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/62fbd8f7f9844ba8b763ba55ac45910b 2024-11-25T10:56:37,368 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/62fbd8f7f9844ba8b763ba55ac45910b, entries=100, sequenceid=441, filesize=9.6 K 2024-11-25T10:56:37,373 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/.tmp/C/38a161f3cf114a10a03a7197c374e850 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/38a161f3cf114a10a03a7197c374e850 2024-11-25T10:56:37,383 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/38a161f3cf114a10a03a7197c374e850, entries=100, sequenceid=441, filesize=9.6 K 2024-11-25T10:56:37,384 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for d938fc35c4eb636e5192d194067fd994 in 779ms, sequenceid=441, compaction requested=true 2024-11-25T10:56:37,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/85a1d960b08742d0b00812ee5ba58a86, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0b2423c49a9740d9b1aac8cf0ba3165c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/764b6ec81093483c95ab904136827094, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/83f543e6ba264ea499da75f829896744, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/87921f7754f34600813f2589d81d760c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/789fed84b586484980a506d6936154a0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0ffbb8e1f8d14f47a4bc613d62c46f80, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/189c3cbfb03d4239ad77ae4ee8529aa9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/5d09f5d479604062b2158afb95a450c2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/72855b411a084bc3ba36bc0f72135489, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7fc98d128a124485b0e016b0f183a2df, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/24478366887745e9b47edfab0b1600b2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b09a83318fd94eb4b1298a1dbc9f10f4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1585d3284bac4149a1570d7ca51cd665, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/d9b8456837ec473290037dcec55c5e5d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6b44a97a59be4940969ed28cb8141545, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/10381fd6cc43497191d7142f7fd665b3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1ef82d41f6c14f0e87e0721e628fa259, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9c2aea282f604a71bc89435f65b981c4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/364e77fe757e41688a4196afadf83bde, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6e5ccb6c6c284ce58027fe76422387d4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3f1d64634f36429f83b1f6b907e6e4bb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/4bdfaa51743a47bd9fa875f1ace94cb8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/488bd1cee4994318aed203ec83823ee8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0498ae2e24bb475eaefdc3b4fcce2e95, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3d06d5f393414699ae5db77c1c865097, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e1e1aae829f8475a8e603a22cadc004d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0dc2953ec8b149da8ca623423701f889, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9a3f14067d314dd3aa82d24ee92c9f73, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e160cc68dfce4f9caed5c318deaab068, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/670aa7c06bda4891adceada8ccca4a07] to archive 2024-11-25T10:56:37,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:56:37,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/85a1d960b08742d0b00812ee5ba58a86 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/85a1d960b08742d0b00812ee5ba58a86 2024-11-25T10:56:37,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0b2423c49a9740d9b1aac8cf0ba3165c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0b2423c49a9740d9b1aac8cf0ba3165c 2024-11-25T10:56:37,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/764b6ec81093483c95ab904136827094 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/764b6ec81093483c95ab904136827094 2024-11-25T10:56:37,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/83f543e6ba264ea499da75f829896744 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/83f543e6ba264ea499da75f829896744 2024-11-25T10:56:37,466 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T10:56:37,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/87921f7754f34600813f2589d81d760c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/87921f7754f34600813f2589d81d760c 2024-11-25T10:56:37,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/789fed84b586484980a506d6936154a0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/789fed84b586484980a506d6936154a0 2024-11-25T10:56:37,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0ffbb8e1f8d14f47a4bc613d62c46f80 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0ffbb8e1f8d14f47a4bc613d62c46f80 2024-11-25T10:56:37,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/189c3cbfb03d4239ad77ae4ee8529aa9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/189c3cbfb03d4239ad77ae4ee8529aa9 2024-11-25T10:56:37,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/5d09f5d479604062b2158afb95a450c2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/5d09f5d479604062b2158afb95a450c2 2024-11-25T10:56:37,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/72855b411a084bc3ba36bc0f72135489 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/72855b411a084bc3ba36bc0f72135489 2024-11-25T10:56:37,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7fc98d128a124485b0e016b0f183a2df to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7fc98d128a124485b0e016b0f183a2df 2024-11-25T10:56:37,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/24478366887745e9b47edfab0b1600b2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/24478366887745e9b47edfab0b1600b2 2024-11-25T10:56:37,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b09a83318fd94eb4b1298a1dbc9f10f4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b09a83318fd94eb4b1298a1dbc9f10f4 2024-11-25T10:56:37,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1585d3284bac4149a1570d7ca51cd665 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1585d3284bac4149a1570d7ca51cd665 2024-11-25T10:56:37,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/d9b8456837ec473290037dcec55c5e5d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/d9b8456837ec473290037dcec55c5e5d 2024-11-25T10:56:37,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6b44a97a59be4940969ed28cb8141545 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6b44a97a59be4940969ed28cb8141545 2024-11-25T10:56:37,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/10381fd6cc43497191d7142f7fd665b3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/10381fd6cc43497191d7142f7fd665b3 2024-11-25T10:56:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T10:56:37,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1ef82d41f6c14f0e87e0721e628fa259 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1ef82d41f6c14f0e87e0721e628fa259 2024-11-25T10:56:37,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9c2aea282f604a71bc89435f65b981c4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9c2aea282f604a71bc89435f65b981c4 2024-11-25T10:56:37,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/364e77fe757e41688a4196afadf83bde to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/364e77fe757e41688a4196afadf83bde 2024-11-25T10:56:37,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6e5ccb6c6c284ce58027fe76422387d4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/6e5ccb6c6c284ce58027fe76422387d4 2024-11-25T10:56:37,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3f1d64634f36429f83b1f6b907e6e4bb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3f1d64634f36429f83b1f6b907e6e4bb 2024-11-25T10:56:37,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/4bdfaa51743a47bd9fa875f1ace94cb8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/4bdfaa51743a47bd9fa875f1ace94cb8 2024-11-25T10:56:37,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/488bd1cee4994318aed203ec83823ee8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/488bd1cee4994318aed203ec83823ee8 2024-11-25T10:56:37,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0498ae2e24bb475eaefdc3b4fcce2e95 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0498ae2e24bb475eaefdc3b4fcce2e95 2024-11-25T10:56:37,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3d06d5f393414699ae5db77c1c865097 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/3d06d5f393414699ae5db77c1c865097 2024-11-25T10:56:37,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e1e1aae829f8475a8e603a22cadc004d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e1e1aae829f8475a8e603a22cadc004d 2024-11-25T10:56:37,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0dc2953ec8b149da8ca623423701f889 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/0dc2953ec8b149da8ca623423701f889 2024-11-25T10:56:37,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9a3f14067d314dd3aa82d24ee92c9f73 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/9a3f14067d314dd3aa82d24ee92c9f73 2024-11-25T10:56:37,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e160cc68dfce4f9caed5c318deaab068 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/e160cc68dfce4f9caed5c318deaab068 2024-11-25T10:56:37,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/670aa7c06bda4891adceada8ccca4a07 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/670aa7c06bda4891adceada8ccca4a07 2024-11-25T10:56:37,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/05ff2790000b4cd2887a3e132a18b0f1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/bbec0da6633c47498ebaf1e87d4f706f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e5b7e98312d2465da58375c2d2f359cf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a631d75de6624a98bf46aecd5e266fe6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/be40d3cfa74c42f690adfdb3d94a7e12, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/da78fab65ba845099cbdaf78431d8ffc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9c02192ec47949bd8f02d86d4620e2e5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/291072b6a1114b488115a8011aa82d4a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/8111fb61eb184fb990fc9701689ae44a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0b2a0aecc10f411d8a994dcda17f905e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/45c8da069a204d7eb5bcebb40c729f2f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/4f2015fe3adc41789ae39f9a59db8c9f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/6a569c10b22f4c24b9f9384101f53a7c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2e40abeeb58342b093d96028778aaf16, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/5eee7d452cb342cf940a383e60af81aa, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a0be72a12fed45e486d82a8fb6a4e4d6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/34b4ff868f004bad9816813de7d8fef4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a542e86e6d93410da8db612328a09536, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e73e43efce7d4348b7d9ba7501ea8202, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/14094819984a499789977cc1fb0de247, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/560cfbfbd58743e38208ded2ea49900d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/d498e3a09c14406aa379775bc89f89cb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c88c518c99a4d69ab1b15501c566533, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/66b22c6129fe44c3a063f28c90b0232e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0cd7273cf3ee4c5685da371403dc1abf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c75ac853e664110bd9b13e514372107, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a22091b19e3444b1b2757318d3945c78, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9e77e4467eae4e5082d55d40425a7f60, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/15f21f6f0ff74b50a07de7e3d2af81fe, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2710204ab91f4a579e5a08c32e3b2fb6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1c8e53b6e1dc49bbaab812ad959e1c5e] to archive 2024-11-25T10:56:37,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:56:37,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/05ff2790000b4cd2887a3e132a18b0f1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/05ff2790000b4cd2887a3e132a18b0f1 2024-11-25T10:56:37,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/bbec0da6633c47498ebaf1e87d4f706f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/bbec0da6633c47498ebaf1e87d4f706f 2024-11-25T10:56:38,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e5b7e98312d2465da58375c2d2f359cf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e5b7e98312d2465da58375c2d2f359cf 2024-11-25T10:56:38,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a631d75de6624a98bf46aecd5e266fe6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a631d75de6624a98bf46aecd5e266fe6 2024-11-25T10:56:38,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/be40d3cfa74c42f690adfdb3d94a7e12 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/be40d3cfa74c42f690adfdb3d94a7e12 2024-11-25T10:56:38,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/da78fab65ba845099cbdaf78431d8ffc to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/da78fab65ba845099cbdaf78431d8ffc 2024-11-25T10:56:38,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9c02192ec47949bd8f02d86d4620e2e5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9c02192ec47949bd8f02d86d4620e2e5 2024-11-25T10:56:38,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/291072b6a1114b488115a8011aa82d4a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/291072b6a1114b488115a8011aa82d4a 2024-11-25T10:56:38,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/8111fb61eb184fb990fc9701689ae44a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/8111fb61eb184fb990fc9701689ae44a 2024-11-25T10:56:38,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0b2a0aecc10f411d8a994dcda17f905e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0b2a0aecc10f411d8a994dcda17f905e 2024-11-25T10:56:38,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/45c8da069a204d7eb5bcebb40c729f2f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/45c8da069a204d7eb5bcebb40c729f2f 2024-11-25T10:56:38,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/4f2015fe3adc41789ae39f9a59db8c9f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/4f2015fe3adc41789ae39f9a59db8c9f 2024-11-25T10:56:38,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/6a569c10b22f4c24b9f9384101f53a7c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/6a569c10b22f4c24b9f9384101f53a7c 2024-11-25T10:56:38,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2e40abeeb58342b093d96028778aaf16 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2e40abeeb58342b093d96028778aaf16 2024-11-25T10:56:38,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/5eee7d452cb342cf940a383e60af81aa to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/5eee7d452cb342cf940a383e60af81aa 2024-11-25T10:56:38,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a0be72a12fed45e486d82a8fb6a4e4d6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a0be72a12fed45e486d82a8fb6a4e4d6 2024-11-25T10:56:38,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/34b4ff868f004bad9816813de7d8fef4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/34b4ff868f004bad9816813de7d8fef4 2024-11-25T10:56:38,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a542e86e6d93410da8db612328a09536 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a542e86e6d93410da8db612328a09536 2024-11-25T10:56:38,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e73e43efce7d4348b7d9ba7501ea8202 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/e73e43efce7d4348b7d9ba7501ea8202 2024-11-25T10:56:38,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/14094819984a499789977cc1fb0de247 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/14094819984a499789977cc1fb0de247 2024-11-25T10:56:38,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/560cfbfbd58743e38208ded2ea49900d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/560cfbfbd58743e38208ded2ea49900d 2024-11-25T10:56:38,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/d498e3a09c14406aa379775bc89f89cb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/d498e3a09c14406aa379775bc89f89cb 2024-11-25T10:56:38,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c88c518c99a4d69ab1b15501c566533 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c88c518c99a4d69ab1b15501c566533 2024-11-25T10:56:38,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/66b22c6129fe44c3a063f28c90b0232e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/66b22c6129fe44c3a063f28c90b0232e 2024-11-25T10:56:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0cd7273cf3ee4c5685da371403dc1abf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0cd7273cf3ee4c5685da371403dc1abf 2024-11-25T10:56:38,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c75ac853e664110bd9b13e514372107 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/0c75ac853e664110bd9b13e514372107 2024-11-25T10:56:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a22091b19e3444b1b2757318d3945c78 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/a22091b19e3444b1b2757318d3945c78 2024-11-25T10:56:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9e77e4467eae4e5082d55d40425a7f60 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/9e77e4467eae4e5082d55d40425a7f60 2024-11-25T10:56:38,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/15f21f6f0ff74b50a07de7e3d2af81fe to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/15f21f6f0ff74b50a07de7e3d2af81fe 2024-11-25T10:56:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2710204ab91f4a579e5a08c32e3b2fb6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/2710204ab91f4a579e5a08c32e3b2fb6 2024-11-25T10:56:38,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1c8e53b6e1dc49bbaab812ad959e1c5e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1c8e53b6e1dc49bbaab812ad959e1c5e 2024-11-25T10:56:38,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/17ab1a97868f441ebaf5cf2a9a618eb2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/839d9200836a4d109e4b9adad72f7da0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/15d07a1a797b4dcdb946bbd08195e043, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e0a9961b776141719f2339c97b22dec6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/71eeac57e2a3492fb7e99947669626c9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/212b63a7aff14a8293639c690b8ba716, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f94d129172324fe5b0536b21c04442f8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/66bea01a666a4923b039748fde15c7ae, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/5bbf3946060a45c59a241e7b8affbb75, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ff156b7b31d04833b52eb8f98b2e774a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/41ce56f31e194134a33d70e9b9638fb2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/69bd51f7d89046958c0c43084b336bbf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/d085b0c8f9654c20b80a5ffc1284835c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0e850dbf4dcc40039908921db8634395, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2e3a068bedcb41f48168f9a545647261, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/bd9e6b54b6db454c8b77ad6167132697, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/11fb87b2104d4ca790949877960a02cf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/9309401314f0405681d6e3445fd36ec4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/19086a92101d4a04ae06af96fa88584f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/45c2ae878bf44984895933f6842fabee, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2d7cbc7d82ff4898aa0e250e933028f8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2532bed5f5ee4a08ad00cacf6e856a19, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0628a4a621084c04a9f6f963b47df2e7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/fbade3a9086d4b19a7ee5b8616591a4c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/a66132d290744387b26826501cd677df, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e08b343c93484630bd2c5c49f9245bcf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/be8f0d83be0d4d6582095e3ae171c7c3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0eeca873a5334e1cb75c348e301b422f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/08ce44a46e7e49d2b32c5621adc35f6e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ba940297334a484096e5686c16ead451, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/13b0ee57074b4ba691427f02f7da6014] to archive 2024-11-25T10:56:38,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:56:38,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/17ab1a97868f441ebaf5cf2a9a618eb2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/17ab1a97868f441ebaf5cf2a9a618eb2 2024-11-25T10:56:38,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/839d9200836a4d109e4b9adad72f7da0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/839d9200836a4d109e4b9adad72f7da0 2024-11-25T10:56:38,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/15d07a1a797b4dcdb946bbd08195e043 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/15d07a1a797b4dcdb946bbd08195e043 2024-11-25T10:56:38,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e0a9961b776141719f2339c97b22dec6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e0a9961b776141719f2339c97b22dec6 2024-11-25T10:56:38,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/71eeac57e2a3492fb7e99947669626c9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/71eeac57e2a3492fb7e99947669626c9 2024-11-25T10:56:38,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/212b63a7aff14a8293639c690b8ba716 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/212b63a7aff14a8293639c690b8ba716 2024-11-25T10:56:38,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f94d129172324fe5b0536b21c04442f8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f94d129172324fe5b0536b21c04442f8 2024-11-25T10:56:38,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/66bea01a666a4923b039748fde15c7ae to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/66bea01a666a4923b039748fde15c7ae 2024-11-25T10:56:38,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/5bbf3946060a45c59a241e7b8affbb75 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/5bbf3946060a45c59a241e7b8affbb75 2024-11-25T10:56:38,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ff156b7b31d04833b52eb8f98b2e774a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ff156b7b31d04833b52eb8f98b2e774a 2024-11-25T10:56:38,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/41ce56f31e194134a33d70e9b9638fb2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/41ce56f31e194134a33d70e9b9638fb2 2024-11-25T10:56:38,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/69bd51f7d89046958c0c43084b336bbf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/69bd51f7d89046958c0c43084b336bbf 2024-11-25T10:56:38,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/d085b0c8f9654c20b80a5ffc1284835c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/d085b0c8f9654c20b80a5ffc1284835c 2024-11-25T10:56:38,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0e850dbf4dcc40039908921db8634395 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0e850dbf4dcc40039908921db8634395 2024-11-25T10:56:38,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2e3a068bedcb41f48168f9a545647261 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2e3a068bedcb41f48168f9a545647261 2024-11-25T10:56:38,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/bd9e6b54b6db454c8b77ad6167132697 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/bd9e6b54b6db454c8b77ad6167132697 2024-11-25T10:56:38,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/11fb87b2104d4ca790949877960a02cf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/11fb87b2104d4ca790949877960a02cf 2024-11-25T10:56:38,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/9309401314f0405681d6e3445fd36ec4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/9309401314f0405681d6e3445fd36ec4 2024-11-25T10:56:38,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/19086a92101d4a04ae06af96fa88584f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/19086a92101d4a04ae06af96fa88584f 2024-11-25T10:56:38,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/45c2ae878bf44984895933f6842fabee to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/45c2ae878bf44984895933f6842fabee 2024-11-25T10:56:38,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2d7cbc7d82ff4898aa0e250e933028f8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2d7cbc7d82ff4898aa0e250e933028f8 2024-11-25T10:56:38,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2532bed5f5ee4a08ad00cacf6e856a19 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/2532bed5f5ee4a08ad00cacf6e856a19 2024-11-25T10:56:38,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0628a4a621084c04a9f6f963b47df2e7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0628a4a621084c04a9f6f963b47df2e7 2024-11-25T10:56:38,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/fbade3a9086d4b19a7ee5b8616591a4c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/fbade3a9086d4b19a7ee5b8616591a4c 2024-11-25T10:56:38,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/a66132d290744387b26826501cd677df to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/a66132d290744387b26826501cd677df 2024-11-25T10:56:38,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e08b343c93484630bd2c5c49f9245bcf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/e08b343c93484630bd2c5c49f9245bcf 2024-11-25T10:56:38,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/be8f0d83be0d4d6582095e3ae171c7c3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/be8f0d83be0d4d6582095e3ae171c7c3 2024-11-25T10:56:38,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0eeca873a5334e1cb75c348e301b422f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/0eeca873a5334e1cb75c348e301b422f 2024-11-25T10:56:38,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/08ce44a46e7e49d2b32c5621adc35f6e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/08ce44a46e7e49d2b32c5621adc35f6e 2024-11-25T10:56:38,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ba940297334a484096e5686c16ead451 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/ba940297334a484096e5686c16ead451 2024-11-25T10:56:38,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/13b0ee57074b4ba691427f02f7da6014 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/13b0ee57074b4ba691427f02f7da6014 2024-11-25T10:56:38,425 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/recovered.edits/444.seqid, newMaxSeqId=444, maxSeqId=1 2024-11-25T10:56:38,426 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. 2024-11-25T10:56:38,426 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for d938fc35c4eb636e5192d194067fd994: Waiting for close lock at 1732532196605Running coprocessor pre-close hooks at 1732532196605Disabling compacts and flushes for region at 1732532196605Disabling writes for close at 1732532196605Obtaining lock to block concurrent updates at 1732532196605Preparing flush snapshotting stores in d938fc35c4eb636e5192d194067fd994 at 1732532196605Finished memstore snapshotting TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732532196606 (+1 ms)Flushing stores of TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994. at 1732532196612 (+6 ms)Flushing d938fc35c4eb636e5192d194067fd994/A: creating writer at 1732532196612Flushing d938fc35c4eb636e5192d194067fd994/A: appending metadata at 1732532196629 (+17 ms)Flushing d938fc35c4eb636e5192d194067fd994/A: closing flushed file at 1732532196629Flushing d938fc35c4eb636e5192d194067fd994/B: creating writer at 1732532196697 (+68 ms)Flushing d938fc35c4eb636e5192d194067fd994/B: appending metadata at 1732532196705 (+8 ms)Flushing d938fc35c4eb636e5192d194067fd994/B: closing flushed file at 1732532196705Flushing d938fc35c4eb636e5192d194067fd994/C: creating writer at 1732532196757 (+52 ms)Flushing d938fc35c4eb636e5192d194067fd994/C: appending metadata at 1732532196764 (+7 ms)Flushing d938fc35c4eb636e5192d194067fd994/C: closing flushed file at 1732532196764Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1edd0228: reopening flushed file at 1732532197236 (+472 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49fb77de: reopening flushed file at 1732532197297 (+61 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53a63873: reopening flushed file at 1732532197368 (+71 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for d938fc35c4eb636e5192d194067fd994 in 779ms, sequenceid=441, compaction requested=true at 1732532197384 (+16 ms)Writing region close event to WAL at 1732532198417 (+1033 ms)Running coprocessor post-close hooks at 1732532198426 (+9 ms)Closed at 1732532198426 2024-11-25T10:56:38,429 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:38,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=d938fc35c4eb636e5192d194067fd994, regionState=CLOSED 2024-11-25T10:56:38,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:56:38,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-25T10:56:38,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure d938fc35c4eb636e5192d194067fd994, server=6bfc37bad115,44671,1732532110409 in 2.0040 sec 2024-11-25T10:56:38,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-25T10:56:38,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d938fc35c4eb636e5192d194067fd994, UNASSIGN in 2.0240 sec 2024-11-25T10:56:38,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-25T10:56:38,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0330 sec 2024-11-25T10:56:38,454 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532198453"}]},"ts":"1732532198453"} 2024-11-25T10:56:38,456 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T10:56:38,456 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T10:56:38,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0820 sec 2024-11-25T10:56:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T10:56:38,557 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:38,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T10:56:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:38,565 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T10:56:38,566 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:38,571 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:38,574 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/recovered.edits] 2024-11-25T10:56:38,577 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1e9fec1845c74f17a42e610f442b6ec3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/1e9fec1845c74f17a42e610f442b6ec3 2024-11-25T10:56:38,579 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7d5f39d1fdef434596fa77c337c5d152 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/7d5f39d1fdef434596fa77c337c5d152 2024-11-25T10:56:38,581 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b1620146841d47a282119e94b50eb34d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/A/b1620146841d47a282119e94b50eb34d 2024-11-25T10:56:38,612 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1ad67637ed8d4b1dbd76d79ce9cb08fd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/1ad67637ed8d4b1dbd76d79ce9cb08fd 2024-11-25T10:56:38,619 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/62fbd8f7f9844ba8b763ba55ac45910b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/62fbd8f7f9844ba8b763ba55ac45910b 2024-11-25T10:56:38,624 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/86700884855a42d7b436d032d0425fbc to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/B/86700884855a42d7b436d032d0425fbc 2024-11-25T10:56:38,629 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/38a161f3cf114a10a03a7197c374e850 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/38a161f3cf114a10a03a7197c374e850 2024-11-25T10:56:38,631 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/8d35ce9ea8dc4f4e983e3695bc842f0d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/8d35ce9ea8dc4f4e983e3695bc842f0d 2024-11-25T10:56:38,634 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f95370670086406cb10d4acb89958c9c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/C/f95370670086406cb10d4acb89958c9c 2024-11-25T10:56:38,656 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/recovered.edits/444.seqid to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994/recovered.edits/444.seqid 2024-11-25T10:56:38,657 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/d938fc35c4eb636e5192d194067fd994 2024-11-25T10:56:38,657 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T10:56:38,660 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:38,664 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T10:56:38,667 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T10:56:38,668 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:38,668 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T10:56:38,669 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732532198668"}]},"ts":"9223372036854775807"} 2024-11-25T10:56:38,671 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T10:56:38,671 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d938fc35c4eb636e5192d194067fd994, NAME => 'TestAcidGuarantees,,1732532171442.d938fc35c4eb636e5192d194067fd994.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T10:56:38,671 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T10:56:38,671 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732532198671"}]},"ts":"9223372036854775807"} 2024-11-25T10:56:38,673 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T10:56:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T10:56:38,674 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:38,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-11-25T10:56:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T10:56:38,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T10:56:38,884 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:38,897 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251 (was 251), OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1539 (was 1396) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1213 (was 1751) 2024-11-25T10:56:38,909 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1539, ProcessCount=11, AvailableMemoryMB=1215 2024-11-25T10:56:38,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:56:38,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:56:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:38,922 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T10:56:38,922 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:38,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-25T10:56:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-25T10:56:38,924 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T10:56:38,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742123_1299 (size=1061) 2024-11-25T10:56:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-25T10:56:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-25T10:56:39,369 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:56:39,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742124_1300 (size=53) 2024-11-25T10:56:39,465 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:56:39,465 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5dec7eb6196e9e7152a65489b5594c0b, disabling compactions & flushes 2024-11-25T10:56:39,465 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,465 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,465 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. after waiting 0 ms 2024-11-25T10:56:39,465 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,465 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,465 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5dec7eb6196e9e7152a65489b5594c0b: Waiting for close lock at 1732532199465Disabling compacts and flushes for region at 1732532199465Disabling writes for close at 1732532199465Writing region close event to WAL at 1732532199465Closed at 1732532199465 2024-11-25T10:56:39,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T10:56:39,470 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732532199470"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532199470"}]},"ts":"1732532199470"} 2024-11-25T10:56:39,494 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T10:56:39,500 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T10:56:39,501 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532199501"}]},"ts":"1732532199501"} 2024-11-25T10:56:39,507 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T10:56:39,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, ASSIGN}] 2024-11-25T10:56:39,515 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, ASSIGN 2024-11-25T10:56:39,518 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, ASSIGN; state=OFFLINE, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=false 2024-11-25T10:56:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-25T10:56:39,671 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:39,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, ASSIGN because future has completed 2024-11-25T10:56:39,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:56:39,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T10:56:39,841 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,841 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:56:39,842 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,842 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:56:39,842 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,842 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,843 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,845 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:39,845 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dec7eb6196e9e7152a65489b5594c0b columnFamilyName A 2024-11-25T10:56:39,845 DEBUG [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:39,845 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(327): Store=5dec7eb6196e9e7152a65489b5594c0b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:39,845 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,846 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:39,846 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dec7eb6196e9e7152a65489b5594c0b columnFamilyName B 2024-11-25T10:56:39,846 DEBUG [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:39,847 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(327): Store=5dec7eb6196e9e7152a65489b5594c0b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:39,847 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,848 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:39,848 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dec7eb6196e9e7152a65489b5594c0b columnFamilyName C 2024-11-25T10:56:39,848 DEBUG [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:39,848 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(327): Store=5dec7eb6196e9e7152a65489b5594c0b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:39,849 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,849 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,854 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,854 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,861 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,861 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,861 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:56:39,863 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,865 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:56:39,866 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 5dec7eb6196e9e7152a65489b5594c0b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68008993, jitterRate=0.013412967324256897}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:56:39,866 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:39,867 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 5dec7eb6196e9e7152a65489b5594c0b: Running coprocessor pre-open hook at 1732532199842Writing region info on filesystem at 1732532199842Initializing all the Stores at 1732532199843 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532199843Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532199843Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532199843Cleaning up temporary data from old regions at 1732532199861 (+18 ms)Running coprocessor post-open hooks at 1732532199866 (+5 ms)Region opened successfully at 1732532199867 (+1 ms) 2024-11-25T10:56:39,868 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., pid=81, masterSystemTime=1732532199826 2024-11-25T10:56:39,870 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,870 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:39,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:39,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:56:39,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-25T10:56:39,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 in 200 msec 2024-11-25T10:56:39,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-25T10:56:39,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, ASSIGN in 367 msec 2024-11-25T10:56:39,882 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T10:56:39,883 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532199883"}]},"ts":"1732532199883"} 2024-11-25T10:56:39,885 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T10:56:39,886 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T10:56:39,887 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 975 msec 2024-11-25T10:56:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-25T10:56:40,065 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:40,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:40,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:40,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:40,070 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651e9646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:40,074 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ca7a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:40,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:40,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:40,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:40,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:40,082 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T10:56:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39013,1732532109078 2024-11-25T10:56:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f679a8d 2024-11-25T10:56:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T10:56:40,084 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T10:56:40,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:56:40,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:56:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T10:56:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742125_1301 (size=1097) 2024-11-25T10:56:40,531 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T10:56:40,531 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T10:56:40,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:56:40,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, REOPEN/MOVE}] 2024-11-25T10:56:40,540 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, REOPEN/MOVE 2024-11-25T10:56:40,540 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:40,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, REOPEN/MOVE because future has completed 2024-11-25T10:56:40,542 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:56:40,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:56:40,696 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:40,696 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:56:40,696 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 5dec7eb6196e9e7152a65489b5594c0b, disabling compactions & flushes 2024-11-25T10:56:40,696 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:40,696 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:40,696 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. after waiting 0 ms 2024-11-25T10:56:40,696 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:40,716 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T10:56:40,717 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:40,717 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 5dec7eb6196e9e7152a65489b5594c0b: Waiting for close lock at 1732532200696Running coprocessor pre-close hooks at 1732532200696Disabling compacts and flushes for region at 1732532200696Disabling writes for close at 1732532200696Writing region close event to WAL at 1732532200696Running coprocessor post-close hooks at 1732532200717 (+21 ms)Closed at 1732532200717 2024-11-25T10:56:40,718 WARN [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 5dec7eb6196e9e7152a65489b5594c0b to self. 2024-11-25T10:56:40,721 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:40,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=CLOSED 2024-11-25T10:56:40,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:56:40,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-25T10:56:40,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 in 185 msec 2024-11-25T10:56:40,730 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, REOPEN/MOVE; state=CLOSED, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=true 2024-11-25T10:56:40,885 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:40,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, REOPEN/MOVE because future has completed 2024-11-25T10:56:40,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:56:41,066 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,067 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:56:41,067 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,067 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:56:41,067 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,067 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,080 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,089 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:41,089 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dec7eb6196e9e7152a65489b5594c0b columnFamilyName A 2024-11-25T10:56:41,100 DEBUG [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:41,105 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(327): Store=5dec7eb6196e9e7152a65489b5594c0b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:41,105 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,108 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:41,108 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dec7eb6196e9e7152a65489b5594c0b columnFamilyName B 2024-11-25T10:56:41,108 DEBUG [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:41,108 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(327): Store=5dec7eb6196e9e7152a65489b5594c0b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:41,109 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,109 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:56:41,109 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dec7eb6196e9e7152a65489b5594c0b columnFamilyName C 2024-11-25T10:56:41,109 DEBUG [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:41,110 INFO [StoreOpener-5dec7eb6196e9e7152a65489b5594c0b-1 {}] regionserver.HStore(327): Store=5dec7eb6196e9e7152a65489b5594c0b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:56:41,110 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,110 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,112 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,115 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,118 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,118 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,119 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:56:41,122 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,124 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 5dec7eb6196e9e7152a65489b5594c0b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73879842, jitterRate=0.10089543461799622}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:56:41,124 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,125 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 5dec7eb6196e9e7152a65489b5594c0b: Running coprocessor pre-open hook at 1732532201067Writing region info on filesystem at 1732532201067Initializing all the Stores at 1732532201071 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532201071Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532201076 (+5 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532201076Cleaning up temporary data from old regions at 1732532201118 (+42 ms)Running coprocessor post-open hooks at 1732532201124 (+6 ms)Region opened successfully at 1732532201125 (+1 ms) 2024-11-25T10:56:41,126 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., pid=86, masterSystemTime=1732532201044 2024-11-25T10:56:41,134 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,134 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,135 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=OPEN, openSeqNum=5, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:56:41,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-25T10:56:41,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 in 276 msec 2024-11-25T10:56:41,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-25T10:56:41,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, REOPEN/MOVE in 641 msec 2024-11-25T10:56:41,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-25T10:56:41,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 665 msec 2024-11-25T10:56:41,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.1190 sec 2024-11-25T10:56:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T10:56:41,221 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:41,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,227 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bebc809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cd2a05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,242 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0b2d5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,249 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1384ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,259 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f17d7aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,261 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea01ae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,276 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562ae9d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,284 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7824469b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,294 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97b4587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d05c84c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f81acc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,309 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b3c93b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,317 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7710a28f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3390ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,336 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b339d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,340 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4222746c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,351 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3733b315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,354 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672539d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:56:41,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:56:41,369 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:56:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:56:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:56:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@612eaab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:56:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:56:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:56:41,371 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:56:41,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24271e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:56:41,377 DEBUG [Thread-1406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,377 DEBUG [Thread-1402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,378 DEBUG [Thread-1406 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,380 DEBUG [Thread-1410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,381 DEBUG [Thread-1410 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,381 DEBUG [Thread-1410 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,383 DEBUG [Thread-1397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,384 DEBUG [Thread-1397 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,384 DEBUG [Thread-1397 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,384 DEBUG [Thread-1406 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,386 DEBUG [Thread-1404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,393 DEBUG [Thread-1395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,399 DEBUG [Thread-1393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:41,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-25T10:56:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-25T10:56:41,404 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,405 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:41,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,420 DEBUG [Thread-1399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,421 DEBUG [Thread-1408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,422 DEBUG [Thread-1408 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,422 DEBUG [Thread-1408 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,427 DEBUG [Thread-1391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:56:41,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:56:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:56:41,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:56:41,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:56:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T10:56:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:41,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:41,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:41,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-25T10:56:41,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:41,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:41,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257278891fee0e4b1386948ac9df2ffa84_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532201446/Put/seqid=0 2024-11-25T10:56:41,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742126_1302 (size=12154) 2024-11-25T10:56:41,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532261575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532201556, max timestamp=1732532201608Num uniques 859; 2024-11-25T10:56:41,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:41,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532201556, max timestamp=1732532201608Num uniques 859; 2024-11-25T10:56:41,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:41,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532201556, max timestamp=1732532201608Num uniques 859; 2024-11-25T10:56:41,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:41,637 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:41,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257278891fee0e4b1386948ac9df2ffa84_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257278891fee0e4b1386948ac9df2ffa84_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:41,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f491dbcce1d942f9aa715ec7b42b740c, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:41,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f491dbcce1d942f9aa715ec7b42b740c is 175, key is test_row_0/A:col10/1732532201446/Put/seqid=0 2024-11-25T10:56:41,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:41,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:41,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-25T10:56:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742127_1303 (size=30955) 2024-11-25T10:56:41,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532261734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532261740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532261743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532261745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532261749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532261859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532261867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:41,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:41,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532261867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:41,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532261881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:41,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532261964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:41,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:41,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:42,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:42,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-25T10:56:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532262077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532262085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532262089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532262107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f491dbcce1d942f9aa715ec7b42b740c 2024-11-25T10:56:42,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/52e388c8ac9e49f29a70ed7fab2ff030 is 50, key is test_row_0/B:col10/1732532201446/Put/seqid=0 2024-11-25T10:56:42,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:42,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:42,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742128_1304 (size=12001) 2024-11-25T10:56:42,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/52e388c8ac9e49f29a70ed7fab2ff030 2024-11-25T10:56:42,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9cccc56e41e44b0aa4f5406afae58017 is 50, key is test_row_0/C:col10/1732532201446/Put/seqid=0 2024-11-25T10:56:42,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532262300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742129_1305 (size=12001) 2024-11-25T10:56:42,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:42,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532262404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532262414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532262415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532262424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:42,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-25T10:56:42,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:42,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:42,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:42,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9cccc56e41e44b0aa4f5406afae58017 2024-11-25T10:56:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f491dbcce1d942f9aa715ec7b42b740c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f491dbcce1d942f9aa715ec7b42b740c 2024-11-25T10:56:42,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f491dbcce1d942f9aa715ec7b42b740c, entries=150, sequenceid=16, filesize=30.2 K 2024-11-25T10:56:42,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/52e388c8ac9e49f29a70ed7fab2ff030 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/52e388c8ac9e49f29a70ed7fab2ff030 2024-11-25T10:56:42,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/52e388c8ac9e49f29a70ed7fab2ff030, entries=150, sequenceid=16, filesize=11.7 K 2024-11-25T10:56:42,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9cccc56e41e44b0aa4f5406afae58017 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9cccc56e41e44b0aa4f5406afae58017 2024-11-25T10:56:42,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9cccc56e41e44b0aa4f5406afae58017, entries=150, sequenceid=16, filesize=11.7 K 2024-11-25T10:56:42,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 5dec7eb6196e9e7152a65489b5594c0b in 1331ms, sequenceid=16, compaction requested=false 2024-11-25T10:56:42,793 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T10:56:42,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:42,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-25T10:56:42,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:42,832 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-25T10:56:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:42,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253501dd6ef6484909872ec376b3d60459_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532201737/Put/seqid=0 2024-11-25T10:56:42,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742130_1306 (size=12154) 2024-11-25T10:56:42,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:42,914 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253501dd6ef6484909872ec376b3d60459_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253501dd6ef6484909872ec376b3d60459_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:42,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/51ba17e410f34bfca209e0f7c4cf15d3, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:42,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/51ba17e410f34bfca209e0f7c4cf15d3 is 175, key is test_row_0/A:col10/1732532201737/Put/seqid=0 2024-11-25T10:56:42,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742131_1307 (size=30955) 2024-11-25T10:56:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532262944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532262962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532262964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532262967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:42,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532262979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532263103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532263103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532263103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532263114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532263335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532263335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,369 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/51ba17e410f34bfca209e0f7c4cf15d3 2024-11-25T10:56:43,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532263358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532263358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/4c79148a0cc9498aa3e555174f69d375 is 50, key is test_row_0/B:col10/1732532201737/Put/seqid=0 2024-11-25T10:56:43,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742132_1308 (size=12001) 2024-11-25T10:56:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-25T10:56:43,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532263664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532263664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532263705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532263705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:43,884 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/4c79148a0cc9498aa3e555174f69d375 2024-11-25T10:56:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/601408bd895a4fba9bdf49517843adaf is 50, key is test_row_0/C:col10/1732532201737/Put/seqid=0 2024-11-25T10:56:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742133_1309 (size=12001) 2024-11-25T10:56:44,013 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T10:56:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532264001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:44,021 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in 5dec7eb6196e9e7152a65489b5594c0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:44,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532264184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:44,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532264184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:44,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532264236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:44,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532264236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:44,387 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/601408bd895a4fba9bdf49517843adaf 2024-11-25T10:56:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/51ba17e410f34bfca209e0f7c4cf15d3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/51ba17e410f34bfca209e0f7c4cf15d3 2024-11-25T10:56:44,493 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/51ba17e410f34bfca209e0f7c4cf15d3, entries=150, sequenceid=50, filesize=30.2 K 2024-11-25T10:56:44,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/4c79148a0cc9498aa3e555174f69d375 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4c79148a0cc9498aa3e555174f69d375 2024-11-25T10:56:44,540 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4c79148a0cc9498aa3e555174f69d375, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T10:56:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/601408bd895a4fba9bdf49517843adaf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/601408bd895a4fba9bdf49517843adaf 2024-11-25T10:56:44,572 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/601408bd895a4fba9bdf49517843adaf, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T10:56:44,577 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 5dec7eb6196e9e7152a65489b5594c0b in 1745ms, sequenceid=50, compaction requested=false 2024-11-25T10:56:44,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:44,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:44,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-25T10:56:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-25T10:56:44,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-25T10:56:44,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1830 sec 2024-11-25T10:56:44,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.2050 sec 2024-11-25T10:56:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:45,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:56:45,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:45,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:45,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:45,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:45,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:45,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:45,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258fd3eb0704c1454aaf3fc482246a0290_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532202962/Put/seqid=0 2024-11-25T10:56:45,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742134_1310 (size=14594) 2024-11-25T10:56:45,309 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:45,323 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258fd3eb0704c1454aaf3fc482246a0290_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258fd3eb0704c1454aaf3fc482246a0290_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:45,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/ee17c694226442c2a7f04ded891b343a, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:45,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/ee17c694226442c2a7f04ded891b343a is 175, key is test_row_0/A:col10/1732532202962/Put/seqid=0 2024-11-25T10:56:45,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532265306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532265319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532265321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532265323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742135_1311 (size=39549) 2024-11-25T10:56:45,364 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/ee17c694226442c2a7f04ded891b343a 2024-11-25T10:56:45,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/ebb421ebd6c9409e8ccf3aa65ed14351 is 50, key is test_row_0/B:col10/1732532202962/Put/seqid=0 2024-11-25T10:56:45,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742136_1312 (size=12001) 2024-11-25T10:56:45,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/ebb421ebd6c9409e8ccf3aa65ed14351 2024-11-25T10:56:45,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f0cc9dda35ee45b5a1603baecb4ddb51 is 50, key is test_row_0/C:col10/1732532202962/Put/seqid=0 2024-11-25T10:56:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532265434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532265434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532265445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532265445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742137_1313 (size=12001) 2024-11-25T10:56:45,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f0cc9dda35ee45b5a1603baecb4ddb51 2024-11-25T10:56:45,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/ee17c694226442c2a7f04ded891b343a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee17c694226442c2a7f04ded891b343a 2024-11-25T10:56:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-25T10:56:45,589 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:45,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:45,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee17c694226442c2a7f04ded891b343a, entries=200, sequenceid=64, filesize=38.6 K 2024-11-25T10:56:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-25T10:56:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T10:56:45,607 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:45,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/ebb421ebd6c9409e8ccf3aa65ed14351 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ebb421ebd6c9409e8ccf3aa65ed14351 2024-11-25T10:56:45,612 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:45,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:45,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ebb421ebd6c9409e8ccf3aa65ed14351, entries=150, sequenceid=64, filesize=11.7 K 2024-11-25T10:56:45,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f0cc9dda35ee45b5a1603baecb4ddb51 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f0cc9dda35ee45b5a1603baecb4ddb51 2024-11-25T10:56:45,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532265657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532265657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532265669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532265677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:45,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f0cc9dda35ee45b5a1603baecb4ddb51, entries=150, sequenceid=64, filesize=11.7 K 2024-11-25T10:56:45,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5dec7eb6196e9e7152a65489b5594c0b in 466ms, sequenceid=64, compaction requested=true 2024-11-25T10:56:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:45,693 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:45,693 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:45,701 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:45,701 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:45,701 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/A is initiating minor compaction (all files) 2024-11-25T10:56:45,701 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/B is initiating minor compaction (all files) 2024-11-25T10:56:45,701 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/A in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:45,701 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/B in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:45,701 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/52e388c8ac9e49f29a70ed7fab2ff030, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4c79148a0cc9498aa3e555174f69d375, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ebb421ebd6c9409e8ccf3aa65ed14351] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=35.2 K 2024-11-25T10:56:45,701 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f491dbcce1d942f9aa715ec7b42b740c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/51ba17e410f34bfca209e0f7c4cf15d3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee17c694226442c2a7f04ded891b343a] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=99.1 K 2024-11-25T10:56:45,701 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:45,701 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f491dbcce1d942f9aa715ec7b42b740c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/51ba17e410f34bfca209e0f7c4cf15d3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee17c694226442c2a7f04ded891b343a] 2024-11-25T10:56:45,705 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e388c8ac9e49f29a70ed7fab2ff030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732532201446 2024-11-25T10:56:45,705 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f491dbcce1d942f9aa715ec7b42b740c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732532201446 2024-11-25T10:56:45,709 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ba17e410f34bfca209e0f7c4cf15d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732532201705 2024-11-25T10:56:45,710 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c79148a0cc9498aa3e555174f69d375, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732532201705 2024-11-25T10:56:45,713 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb421ebd6c9409e8ccf3aa65ed14351, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732532202962 2024-11-25T10:56:45,713 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ee17c694226442c2a7f04ded891b343a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732532202919 2024-11-25T10:56:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T10:56:45,757 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:45,775 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:45,776 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/ad0cf5b2b9d945d1aa69b4a1d6475202 is 50, key is test_row_0/B:col10/1732532202962/Put/seqid=0 2024-11-25T10:56:45,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-25T10:56:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:45,781 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:56:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:45,783 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112553bf568eb1cb4325899d266f9888747e_5dec7eb6196e9e7152a65489b5594c0b store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:45,786 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112553bf568eb1cb4325899d266f9888747e_5dec7eb6196e9e7152a65489b5594c0b, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:45,786 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112553bf568eb1cb4325899d266f9888747e_5dec7eb6196e9e7152a65489b5594c0b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:45,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fc8a059911654a689682655209a0bee5_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532205284/Put/seqid=0 2024-11-25T10:56:45,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742138_1314 (size=12104) 2024-11-25T10:56:45,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742140_1316 (size=12154) 2024-11-25T10:56:45,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:45,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742139_1315 (size=4469) 2024-11-25T10:56:45,908 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fc8a059911654a689682655209a0bee5_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125fc8a059911654a689682655209a0bee5_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:45,909 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/ad0cf5b2b9d945d1aa69b4a1d6475202 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ad0cf5b2b9d945d1aa69b4a1d6475202 2024-11-25T10:56:45,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/5c3af7ef17634d558e9836a593fda9a7, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:45,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/5c3af7ef17634d558e9836a593fda9a7 is 175, key is test_row_0/A:col10/1732532205284/Put/seqid=0 2024-11-25T10:56:45,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742141_1317 (size=30955) 2024-11-25T10:56:45,919 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/5c3af7ef17634d558e9836a593fda9a7 2024-11-25T10:56:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T10:56:45,934 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/B of 5dec7eb6196e9e7152a65489b5594c0b into ad0cf5b2b9d945d1aa69b4a1d6475202(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:45,934 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:45,934 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/B, priority=13, startTime=1732532205693; duration=0sec 2024-11-25T10:56:45,934 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:45,934 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:B 2024-11-25T10:56:45,934 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:45,935 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:45,935 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/C is initiating minor compaction (all files) 2024-11-25T10:56:45,935 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/C in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:45,935 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9cccc56e41e44b0aa4f5406afae58017, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/601408bd895a4fba9bdf49517843adaf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f0cc9dda35ee45b5a1603baecb4ddb51] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=35.2 K 2024-11-25T10:56:45,935 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cccc56e41e44b0aa4f5406afae58017, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732532201446 2024-11-25T10:56:45,936 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 601408bd895a4fba9bdf49517843adaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732532201705 2024-11-25T10:56:45,936 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0cc9dda35ee45b5a1603baecb4ddb51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732532202962 2024-11-25T10:56:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/c3e5288617ff4039b26dfc54446833b3 is 50, key is test_row_0/B:col10/1732532205284/Put/seqid=0 2024-11-25T10:56:45,947 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#C#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:45,947 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/c5dc017594854d21ac2fdfe8611dd493 is 50, key is test_row_0/C:col10/1732532202962/Put/seqid=0 2024-11-25T10:56:45,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742142_1318 (size=12001) 2024-11-25T10:56:45,978 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/c3e5288617ff4039b26dfc54446833b3 2024-11-25T10:56:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:45,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/eba7592dfc144edc8c9518938345d5d2 is 50, key is test_row_0/C:col10/1732532205284/Put/seqid=0 2024-11-25T10:56:46,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742143_1319 (size=12104) 2024-11-25T10:56:46,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742144_1320 (size=12001) 2024-11-25T10:56:46,035 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/eba7592dfc144edc8c9518938345d5d2 2024-11-25T10:56:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532266044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532266046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,053 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in 5dec7eb6196e9e7152a65489b5594c0b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4304 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/5c3af7ef17634d558e9836a593fda9a7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/5c3af7ef17634d558e9836a593fda9a7 2024-11-25T10:56:46,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532266048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532266050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532266056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,081 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/5c3af7ef17634d558e9836a593fda9a7, entries=150, sequenceid=82, filesize=30.2 K 2024-11-25T10:56:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/c3e5288617ff4039b26dfc54446833b3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/c3e5288617ff4039b26dfc54446833b3 2024-11-25T10:56:46,093 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/c3e5288617ff4039b26dfc54446833b3, entries=150, sequenceid=82, filesize=11.7 K 2024-11-25T10:56:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/eba7592dfc144edc8c9518938345d5d2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/eba7592dfc144edc8c9518938345d5d2 2024-11-25T10:56:46,102 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/eba7592dfc144edc8c9518938345d5d2, entries=150, sequenceid=82, filesize=11.7 K 2024-11-25T10:56:46,104 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5dec7eb6196e9e7152a65489b5594c0b in 324ms, sequenceid=82, compaction requested=false 2024-11-25T10:56:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-25T10:56:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-25T10:56:46,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-25T10:56:46,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 496 msec 2024-11-25T10:56:46,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 515 msec 2024-11-25T10:56:46,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:56:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:46,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112576b759d4f54741df9a01310f466cc82e_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532206180/Put/seqid=0 2024-11-25T10:56:46,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742145_1321 (size=14594) 2024-11-25T10:56:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-25T10:56:46,235 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:46,236 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-25T10:56:46,238 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:46,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T10:56:46,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:46,308 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#A#compaction#267 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:46,308 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/68b7988eac6a4207b2d414717235030d is 175, key is test_row_0/A:col10/1732532202962/Put/seqid=0 2024-11-25T10:56:46,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742146_1322 (size=31058) 2024-11-25T10:56:46,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532266309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T10:56:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532266321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532266329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532266319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,367 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/68b7988eac6a4207b2d414717235030d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/68b7988eac6a4207b2d414717235030d 2024-11-25T10:56:46,384 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/A of 5dec7eb6196e9e7152a65489b5594c0b into 68b7988eac6a4207b2d414717235030d(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:46,384 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:46,384 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/A, priority=13, startTime=1732532205693; duration=0sec 2024-11-25T10:56:46,384 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:46,384 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:A 2024-11-25T10:56:46,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532266445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,465 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/c5dc017594854d21ac2fdfe8611dd493 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c5dc017594854d21ac2fdfe8611dd493 2024-11-25T10:56:46,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532266457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532266464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532266466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,501 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/C of 5dec7eb6196e9e7152a65489b5594c0b into c5dc017594854d21ac2fdfe8611dd493(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:46,502 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:46,502 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/C, priority=13, startTime=1732532205693; duration=0sec 2024-11-25T10:56:46,502 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:46,502 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:C 2024-11-25T10:56:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T10:56:46,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:46,674 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112576b759d4f54741df9a01310f466cc82e_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112576b759d4f54741df9a01310f466cc82e_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:46,677 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/17338d38aae646599989ca029909673b, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:46,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/17338d38aae646599989ca029909673b is 175, key is test_row_0/A:col10/1732532206180/Put/seqid=0 2024-11-25T10:56:46,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532266664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532266683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532266685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:46,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:46,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532266694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:46,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742147_1323 (size=39549) 2024-11-25T10:56:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T10:56:46,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:46,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:46,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:46,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532266994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532267021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:47,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532267025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532267028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/17338d38aae646599989ca029909673b 2024-11-25T10:56:47,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:47,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:47,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/af494c60fff04fc383b2e84828d5831d is 50, key is test_row_0/B:col10/1732532206180/Put/seqid=0 2024-11-25T10:56:47,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742148_1324 (size=12001) 2024-11-25T10:56:47,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:47,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:47,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T10:56:47,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:47,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532267516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532267540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532267554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532267569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:47,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:47,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/af494c60fff04fc383b2e84828d5831d 2024-11-25T10:56:47,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:47,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:47,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:47,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/54eb8f85791b430ea69a20439bd6f4d9 is 50, key is test_row_0/C:col10/1732532206180/Put/seqid=0 2024-11-25T10:56:47,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742149_1325 (size=12001) 2024-11-25T10:56:47,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/54eb8f85791b430ea69a20439bd6f4d9 2024-11-25T10:56:47,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/17338d38aae646599989ca029909673b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/17338d38aae646599989ca029909673b 2024-11-25T10:56:47,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/17338d38aae646599989ca029909673b, entries=200, sequenceid=98, filesize=38.6 K 2024-11-25T10:56:47,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/af494c60fff04fc383b2e84828d5831d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/af494c60fff04fc383b2e84828d5831d 2024-11-25T10:56:47,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/af494c60fff04fc383b2e84828d5831d, entries=150, sequenceid=98, filesize=11.7 K 2024-11-25T10:56:47,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/54eb8f85791b430ea69a20439bd6f4d9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/54eb8f85791b430ea69a20439bd6f4d9 2024-11-25T10:56:47,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/54eb8f85791b430ea69a20439bd6f4d9, entries=150, sequenceid=98, filesize=11.7 K 2024-11-25T10:56:47,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5dec7eb6196e9e7152a65489b5594c0b in 1626ms, sequenceid=98, compaction requested=true 2024-11-25T10:56:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:47,809 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:47,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:47,810 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:47,810 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/A is initiating minor compaction (all files) 2024-11-25T10:56:47,810 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/A in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,810 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/68b7988eac6a4207b2d414717235030d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/5c3af7ef17634d558e9836a593fda9a7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/17338d38aae646599989ca029909673b] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=99.2 K 2024-11-25T10:56:47,810 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,810 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/68b7988eac6a4207b2d414717235030d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/5c3af7ef17634d558e9836a593fda9a7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/17338d38aae646599989ca029909673b] 2024-11-25T10:56:47,811 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b7988eac6a4207b2d414717235030d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732532202962 2024-11-25T10:56:47,811 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c3af7ef17634d558e9836a593fda9a7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732532205284 2024-11-25T10:56:47,811 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17338d38aae646599989ca029909673b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732532206020 2024-11-25T10:56:47,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:47,812 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:47,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:47,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:47,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:47,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:47,821 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:47,821 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/B is initiating minor compaction (all files) 2024-11-25T10:56:47,821 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/B in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,821 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ad0cf5b2b9d945d1aa69b4a1d6475202, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/c3e5288617ff4039b26dfc54446833b3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/af494c60fff04fc383b2e84828d5831d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=35.3 K 2024-11-25T10:56:47,824 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0cf5b2b9d945d1aa69b4a1d6475202, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732532202962 2024-11-25T10:56:47,825 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e5288617ff4039b26dfc54446833b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732532205284 2024-11-25T10:56:47,825 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting af494c60fff04fc383b2e84828d5831d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732532206020 2024-11-25T10:56:47,833 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:47,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T10:56:47,848 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:47,848 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/8e2fee8f6e6f455a8aa6077b4828e49e is 50, key is test_row_0/B:col10/1732532206180/Put/seqid=0 2024-11-25T10:56:47,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:47,856 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:56:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:47,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:47,858 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411258f3c67495847443c88e138dc2162df76_5dec7eb6196e9e7152a65489b5594c0b store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:47,860 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411258f3c67495847443c88e138dc2162df76_5dec7eb6196e9e7152a65489b5594c0b, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:47,860 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258f3c67495847443c88e138dc2162df76_5dec7eb6196e9e7152a65489b5594c0b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:47,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742150_1326 (size=12207) 2024-11-25T10:56:47,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742151_1327 (size=4469) 2024-11-25T10:56:47,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ae6ad5f52297494fbf6eeb34eb5366ca_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532206300/Put/seqid=0 2024-11-25T10:56:47,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742152_1328 (size=12154) 2024-11-25T10:56:47,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:47,974 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ae6ad5f52297494fbf6eeb34eb5366ca_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae6ad5f52297494fbf6eeb34eb5366ca_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e386e90f693448c2891f229efa67e9ba, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e386e90f693448c2891f229efa67e9ba is 175, key is test_row_0/A:col10/1732532206300/Put/seqid=0 2024-11-25T10:56:48,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742153_1329 (size=30955) 2024-11-25T10:56:48,062 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e386e90f693448c2891f229efa67e9ba 2024-11-25T10:56:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bf960b6fdad5494c8a082efa69421936 is 50, key is test_row_0/B:col10/1732532206300/Put/seqid=0 2024-11-25T10:56:48,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742154_1330 (size=12001) 2024-11-25T10:56:48,119 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bf960b6fdad5494c8a082efa69421936 2024-11-25T10:56:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9eebefe8a524499fa97a14bfad984ff9 is 50, key is test_row_0/C:col10/1732532206300/Put/seqid=0 2024-11-25T10:56:48,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742155_1331 (size=12001) 2024-11-25T10:56:48,345 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#A#compaction#276 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:48,346 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/6f86920482bb4272a312dd166013771d is 175, key is test_row_0/A:col10/1732532206180/Put/seqid=0 2024-11-25T10:56:48,348 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/8e2fee8f6e6f455a8aa6077b4828e49e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/8e2fee8f6e6f455a8aa6077b4828e49e 2024-11-25T10:56:48,373 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/B of 5dec7eb6196e9e7152a65489b5594c0b into 8e2fee8f6e6f455a8aa6077b4828e49e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:48,373 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:48,373 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/B, priority=13, startTime=1732532207812; duration=0sec 2024-11-25T10:56:48,373 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:48,373 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:B 2024-11-25T10:56:48,373 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:48,387 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:48,387 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/C is initiating minor compaction (all files) 2024-11-25T10:56:48,387 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/C in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:48,387 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c5dc017594854d21ac2fdfe8611dd493, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/eba7592dfc144edc8c9518938345d5d2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/54eb8f85791b430ea69a20439bd6f4d9] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=35.3 K 2024-11-25T10:56:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T10:56:48,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742156_1332 (size=31161) 2024-11-25T10:56:48,396 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c5dc017594854d21ac2fdfe8611dd493, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732532202962 2024-11-25T10:56:48,402 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting eba7592dfc144edc8c9518938345d5d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732532205284 2024-11-25T10:56:48,402 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 54eb8f85791b430ea69a20439bd6f4d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732532206020 2024-11-25T10:56:48,419 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:48,420 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/c7a46e1be3e44baeb0ad95d97a609d0c is 50, key is test_row_0/C:col10/1732532206180/Put/seqid=0 2024-11-25T10:56:48,429 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/6f86920482bb4272a312dd166013771d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6f86920482bb4272a312dd166013771d 2024-11-25T10:56:48,445 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/A of 5dec7eb6196e9e7152a65489b5594c0b into 6f86920482bb4272a312dd166013771d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:48,445 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:48,445 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/A, priority=13, startTime=1732532207809; duration=0sec 2024-11-25T10:56:48,445 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:48,445 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:A 2024-11-25T10:56:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742157_1333 (size=12207) 2024-11-25T10:56:48,525 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/c7a46e1be3e44baeb0ad95d97a609d0c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c7a46e1be3e44baeb0ad95d97a609d0c 2024-11-25T10:56:48,561 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/C of 5dec7eb6196e9e7152a65489b5594c0b into c7a46e1be3e44baeb0ad95d97a609d0c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:48,561 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:48,561 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/C, priority=13, startTime=1732532207813; duration=0sec 2024-11-25T10:56:48,562 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:48,562 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:C 2024-11-25T10:56:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:48,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:48,593 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9eebefe8a524499fa97a14bfad984ff9 2024-11-25T10:56:48,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e386e90f693448c2891f229efa67e9ba as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e386e90f693448c2891f229efa67e9ba 2024-11-25T10:56:48,673 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e386e90f693448c2891f229efa67e9ba, entries=150, sequenceid=117, filesize=30.2 K 2024-11-25T10:56:48,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bf960b6fdad5494c8a082efa69421936 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf960b6fdad5494c8a082efa69421936 2024-11-25T10:56:48,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532268661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532268672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:48,698 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf960b6fdad5494c8a082efa69421936, entries=150, sequenceid=117, filesize=11.7 K 2024-11-25T10:56:48,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9eebefe8a524499fa97a14bfad984ff9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9eebefe8a524499fa97a14bfad984ff9 2024-11-25T10:56:48,704 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9eebefe8a524499fa97a14bfad984ff9, entries=150, sequenceid=117, filesize=11.7 K 2024-11-25T10:56:48,710 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5dec7eb6196e9e7152a65489b5594c0b in 854ms, sequenceid=117, compaction requested=false 2024-11-25T10:56:48,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:48,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:48,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-25T10:56:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-25T10:56:48,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-25T10:56:48,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4810 sec 2024-11-25T10:56:48,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.4960 sec 2024-11-25T10:56:48,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:56:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:48,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112540287d5b1c7c4f2691731d09ae18f689_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532208689/Put/seqid=0 2024-11-25T10:56:48,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742158_1334 (size=12304) 2024-11-25T10:56:48,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532268896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:48,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532268900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:48,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532268919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:48,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532268919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532269024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532269024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532269044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532269044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:49,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112540287d5b1c7c4f2691731d09ae18f689_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112540287d5b1c7c4f2691731d09ae18f689_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:49,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f3bb396041264294bce3d1c3e886a0b0, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:49,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f3bb396041264294bce3d1c3e886a0b0 is 175, key is test_row_0/A:col10/1732532208689/Put/seqid=0 2024-11-25T10:56:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532269245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532269247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742159_1335 (size=31105) 2024-11-25T10:56:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532269263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532269264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532269565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532269576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532269606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532269606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:49,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f3bb396041264294bce3d1c3e886a0b0 2024-11-25T10:56:49,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/473f6128a5a943349b1d03abc013059d is 50, key is test_row_0/B:col10/1732532208689/Put/seqid=0 2024-11-25T10:56:49,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742160_1336 (size=12151) 2024-11-25T10:56:49,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/473f6128a5a943349b1d03abc013059d 2024-11-25T10:56:49,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T10:56:49,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T10:56:49,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/aa39291411d84814bbe3945103782f67 is 50, key is test_row_0/C:col10/1732532208689/Put/seqid=0 2024-11-25T10:56:49,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742161_1337 (size=12151) 2024-11-25T10:56:49,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/aa39291411d84814bbe3945103782f67 2024-11-25T10:56:49,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f3bb396041264294bce3d1c3e886a0b0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f3bb396041264294bce3d1c3e886a0b0 2024-11-25T10:56:49,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f3bb396041264294bce3d1c3e886a0b0, entries=150, sequenceid=137, filesize=30.4 K 2024-11-25T10:56:49,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/473f6128a5a943349b1d03abc013059d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/473f6128a5a943349b1d03abc013059d 2024-11-25T10:56:49,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/473f6128a5a943349b1d03abc013059d, entries=150, sequenceid=137, filesize=11.9 K 2024-11-25T10:56:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/aa39291411d84814bbe3945103782f67 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/aa39291411d84814bbe3945103782f67 2024-11-25T10:56:49,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/aa39291411d84814bbe3945103782f67, entries=150, sequenceid=137, filesize=11.9 K 2024-11-25T10:56:49,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 5dec7eb6196e9e7152a65489b5594c0b in 1203ms, sequenceid=137, compaction requested=true 2024-11-25T10:56:49,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:49,941 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:49,941 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:49,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:49,955 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:49,955 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:49,955 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/B is initiating minor compaction (all files) 2024-11-25T10:56:49,955 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/A is initiating minor compaction (all files) 2024-11-25T10:56:49,955 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/B in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:49,955 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/A in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:49,955 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/8e2fee8f6e6f455a8aa6077b4828e49e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf960b6fdad5494c8a082efa69421936, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/473f6128a5a943349b1d03abc013059d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=35.5 K 2024-11-25T10:56:49,955 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6f86920482bb4272a312dd166013771d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e386e90f693448c2891f229efa67e9ba, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f3bb396041264294bce3d1c3e886a0b0] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=91.0 K 2024-11-25T10:56:49,955 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:49,955 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6f86920482bb4272a312dd166013771d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e386e90f693448c2891f229efa67e9ba, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f3bb396041264294bce3d1c3e886a0b0] 2024-11-25T10:56:49,960 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f86920482bb4272a312dd166013771d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732532206020 2024-11-25T10:56:49,960 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e2fee8f6e6f455a8aa6077b4828e49e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732532206020 2024-11-25T10:56:49,965 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e386e90f693448c2891f229efa67e9ba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732532206300 2024-11-25T10:56:49,965 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf960b6fdad5494c8a082efa69421936, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732532206300 2024-11-25T10:56:49,970 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bb396041264294bce3d1c3e886a0b0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732532208689 2024-11-25T10:56:49,970 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 473f6128a5a943349b1d03abc013059d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732532208689 2024-11-25T10:56:50,025 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:50,035 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:50,036 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/828cdc9912a04de0ab1a585e6cfdf4f6 is 50, key is test_row_0/B:col10/1732532208689/Put/seqid=0 2024-11-25T10:56:50,042 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112579fb34016e164094ad444c5c7ccbff42_5dec7eb6196e9e7152a65489b5594c0b store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:50,044 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112579fb34016e164094ad444c5c7ccbff42_5dec7eb6196e9e7152a65489b5594c0b, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:50,044 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112579fb34016e164094ad444c5c7ccbff42_5dec7eb6196e9e7152a65489b5594c0b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:50,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:56:50,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:50,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:50,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:50,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:50,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:50,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:50,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742162_1338 (size=12459) 2024-11-25T10:56:50,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742163_1339 (size=4469) 2024-11-25T10:56:50,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125441ef0cae5ae46029cfd3bafd1e03414_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532208917/Put/seqid=0 2024-11-25T10:56:50,141 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/828cdc9912a04de0ab1a585e6cfdf4f6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/828cdc9912a04de0ab1a585e6cfdf4f6 2024-11-25T10:56:50,169 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/B of 5dec7eb6196e9e7152a65489b5594c0b into 828cdc9912a04de0ab1a585e6cfdf4f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:50,170 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:50,170 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/B, priority=13, startTime=1732532209941; duration=0sec 2024-11-25T10:56:50,170 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:50,170 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:B 2024-11-25T10:56:50,170 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:50,172 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:50,172 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/C is initiating minor compaction (all files) 2024-11-25T10:56:50,172 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/C in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:50,172 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c7a46e1be3e44baeb0ad95d97a609d0c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9eebefe8a524499fa97a14bfad984ff9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/aa39291411d84814bbe3945103782f67] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=35.5 K 2024-11-25T10:56:50,172 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7a46e1be3e44baeb0ad95d97a609d0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732532206020 2024-11-25T10:56:50,173 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eebefe8a524499fa97a14bfad984ff9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732532206300 2024-11-25T10:56:50,173 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa39291411d84814bbe3945103782f67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732532208689 2024-11-25T10:56:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532270164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,208 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:50,209 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/79d9522623d641ed90594b10dd06a5f7 is 50, key is test_row_0/C:col10/1732532208689/Put/seqid=0 2024-11-25T10:56:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532270172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532270181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742164_1340 (size=14794) 2024-11-25T10:56:50,221 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:50,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532270196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532270200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125441ef0cae5ae46029cfd3bafd1e03414_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125441ef0cae5ae46029cfd3bafd1e03414_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:50,238 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a1b50fd7af454013a27a287ea333b61d, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:50,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a1b50fd7af454013a27a287ea333b61d is 175, key is test_row_0/A:col10/1732532208917/Put/seqid=0 2024-11-25T10:56:50,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742165_1341 (size=12459) 2024-11-25T10:56:50,280 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/79d9522623d641ed90594b10dd06a5f7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/79d9522623d641ed90594b10dd06a5f7 2024-11-25T10:56:50,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742166_1342 (size=39749) 2024-11-25T10:56:50,293 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/C of 5dec7eb6196e9e7152a65489b5594c0b into 79d9522623d641ed90594b10dd06a5f7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:50,293 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:50,293 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/C, priority=13, startTime=1732532209941; duration=0sec 2024-11-25T10:56:50,294 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:50,294 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:C 2024-11-25T10:56:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532270307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532270325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532270328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532270345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532270353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T10:56:50,417 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:50,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-25T10:56:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T10:56:50,429 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:50,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:50,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:50,521 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#A#compaction#285 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:50,521 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/8609d2d46db04c4289f72ee490b4c707 is 175, key is test_row_0/A:col10/1732532208689/Put/seqid=0 2024-11-25T10:56:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T10:56:50,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532270546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742167_1343 (size=31413) 2024-11-25T10:56:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532270554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532270563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532270566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532270574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T10:56:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:50,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a1b50fd7af454013a27a287ea333b61d 2024-11-25T10:56:50,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/12b6658fcf8346529ccd417a6d673ceb is 50, key is test_row_0/B:col10/1732532208917/Put/seqid=0 2024-11-25T10:56:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T10:56:50,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T10:56:50,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:50,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:50,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742168_1344 (size=12151) 2024-11-25T10:56:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/12b6658fcf8346529ccd417a6d673ceb 2024-11-25T10:56:50,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/7d03d634d07d4fbc8c928db169bc2274 is 50, key is test_row_0/C:col10/1732532208917/Put/seqid=0 2024-11-25T10:56:50,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742169_1345 (size=12151) 2024-11-25T10:56:50,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/7d03d634d07d4fbc8c928db169bc2274 2024-11-25T10:56:50,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532270874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532270893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a1b50fd7af454013a27a287ea333b61d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a1b50fd7af454013a27a287ea333b61d 2024-11-25T10:56:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532270894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532270894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532270897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:50,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T10:56:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:50,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:50,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a1b50fd7af454013a27a287ea333b61d, entries=200, sequenceid=154, filesize=38.8 K 2024-11-25T10:56:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/12b6658fcf8346529ccd417a6d673ceb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/12b6658fcf8346529ccd417a6d673ceb 2024-11-25T10:56:50,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/12b6658fcf8346529ccd417a6d673ceb, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T10:56:50,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/7d03d634d07d4fbc8c928db169bc2274 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/7d03d634d07d4fbc8c928db169bc2274 2024-11-25T10:56:50,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/7d03d634d07d4fbc8c928db169bc2274, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T10:56:50,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5dec7eb6196e9e7152a65489b5594c0b in 895ms, sequenceid=154, compaction requested=false 2024-11-25T10:56:50,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:50,989 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/8609d2d46db04c4289f72ee490b4c707 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/8609d2d46db04c4289f72ee490b4c707 2024-11-25T10:56:51,035 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/A of 5dec7eb6196e9e7152a65489b5594c0b into 8609d2d46db04c4289f72ee490b4c707(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-25T10:56:51,035 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:51,035 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/A, priority=13, startTime=1732532209941; duration=1sec 2024-11-25T10:56:51,035 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:51,035 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:A 2024-11-25T10:56:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T10:56:51,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T10:56:51,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:51,074 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:56:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:51,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aa0ce553028b4873a186e5c348c0d067_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532210189/Put/seqid=0 2024-11-25T10:56:51,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742170_1346 (size=12304) 2024-11-25T10:56:51,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:51,184 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aa0ce553028b4873a186e5c348c0d067_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aa0ce553028b4873a186e5c348c0d067_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:51,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/add5282f8aec499d80c2b04b2c838217, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:51,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/add5282f8aec499d80c2b04b2c838217 is 175, key is test_row_0/A:col10/1732532210189/Put/seqid=0 2024-11-25T10:56:51,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742171_1347 (size=31105) 2024-11-25T10:56:51,232 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/add5282f8aec499d80c2b04b2c838217 2024-11-25T10:56:51,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/0ca08961cdf94138adb3d22fce78ad5a is 50, key is test_row_0/B:col10/1732532210189/Put/seqid=0 2024-11-25T10:56:51,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742172_1348 (size=12151) 2024-11-25T10:56:51,295 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/0ca08961cdf94138adb3d22fce78ad5a 2024-11-25T10:56:51,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f8c95f2df945476f99b11f762fd78062 is 50, key is test_row_0/C:col10/1732532210189/Put/seqid=0 2024-11-25T10:56:51,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742173_1349 (size=12151) 2024-11-25T10:56:51,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532271471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532271477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532271479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532271506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532271505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T10:56:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532271614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532271614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532271620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532271674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532271675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,769 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f8c95f2df945476f99b11f762fd78062 2024-11-25T10:56:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/add5282f8aec499d80c2b04b2c838217 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/add5282f8aec499d80c2b04b2c838217 2024-11-25T10:56:51,780 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/add5282f8aec499d80c2b04b2c838217, entries=150, sequenceid=172, filesize=30.4 K 2024-11-25T10:56:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/0ca08961cdf94138adb3d22fce78ad5a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/0ca08961cdf94138adb3d22fce78ad5a 2024-11-25T10:56:51,787 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/0ca08961cdf94138adb3d22fce78ad5a, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T10:56:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f8c95f2df945476f99b11f762fd78062 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f8c95f2df945476f99b11f762fd78062 2024-11-25T10:56:51,793 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f8c95f2df945476f99b11f762fd78062, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T10:56:51,803 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 5dec7eb6196e9e7152a65489b5594c0b in 730ms, sequenceid=172, compaction requested=true 2024-11-25T10:56:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-25T10:56:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-25T10:56:51,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-25T10:56:51,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3790 sec 2024-11-25T10:56:51,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.3950 sec 2024-11-25T10:56:51,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:56:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:51,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:51,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112567da03a2703a4e338e8b1abb70edaec2_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532211461/Put/seqid=0 2024-11-25T10:56:51,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742174_1350 (size=14794) 2024-11-25T10:56:51,920 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:51,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532271904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112567da03a2703a4e338e8b1abb70edaec2_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112567da03a2703a4e338e8b1abb70edaec2_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:51,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/2dd5b82e77b44e5fb3aabcdfa45d888f, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:51,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/2dd5b82e77b44e5fb3aabcdfa45d888f is 175, key is test_row_0/A:col10/1732532211461/Put/seqid=0 2024-11-25T10:56:51,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532271909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532271913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532271922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:51,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532271922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:51,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742175_1351 (size=39749) 2024-11-25T10:56:51,955 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/2dd5b82e77b44e5fb3aabcdfa45d888f 2024-11-25T10:56:51,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/7c4a562540e641219a462469b36459d7 is 50, key is test_row_0/B:col10/1732532211461/Put/seqid=0 2024-11-25T10:56:52,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742176_1352 (size=12151) 2024-11-25T10:56:52,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532272037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532272054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532272056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532272057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532272240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532272264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532272289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532272289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532272292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/7c4a562540e641219a462469b36459d7 2024-11-25T10:56:52,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/2bedcf61e19b4d61ac09aa225cb2ee77 is 50, key is test_row_0/C:col10/1732532211461/Put/seqid=0 2024-11-25T10:56:52,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742177_1353 (size=12151) 2024-11-25T10:56:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T10:56:52,601 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:52,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-25T10:56:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T10:56:52,625 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:52,627 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:52,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:52,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532272617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532272625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532272625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532272625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T10:56:52,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T10:56:52,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:52,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:52,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:52,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532272774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:52,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/2bedcf61e19b4d61ac09aa225cb2ee77 2024-11-25T10:56:52,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/2dd5b82e77b44e5fb3aabcdfa45d888f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2dd5b82e77b44e5fb3aabcdfa45d888f 2024-11-25T10:56:52,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2dd5b82e77b44e5fb3aabcdfa45d888f, entries=200, sequenceid=191, filesize=38.8 K 2024-11-25T10:56:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/7c4a562540e641219a462469b36459d7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7c4a562540e641219a462469b36459d7 2024-11-25T10:56:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T10:56:52,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T10:56:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:52,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:52,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7c4a562540e641219a462469b36459d7, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T10:56:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:52,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/2bedcf61e19b4d61ac09aa225cb2ee77 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2bedcf61e19b4d61ac09aa225cb2ee77 2024-11-25T10:56:52,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2bedcf61e19b4d61ac09aa225cb2ee77, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T10:56:52,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5dec7eb6196e9e7152a65489b5594c0b in 1095ms, sequenceid=191, compaction requested=true 2024-11-25T10:56:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:56:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:52,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:56:52,961 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:52,962 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:52,969 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:52,969 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/C is initiating minor compaction (all files) 2024-11-25T10:56:52,969 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/C in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:52,969 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/79d9522623d641ed90594b10dd06a5f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/7d03d634d07d4fbc8c928db169bc2274, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f8c95f2df945476f99b11f762fd78062, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2bedcf61e19b4d61ac09aa225cb2ee77] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=47.8 K 2024-11-25T10:56:52,970 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:52,970 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/A is initiating minor compaction (all files) 2024-11-25T10:56:52,970 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/A in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:52,970 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/8609d2d46db04c4289f72ee490b4c707, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a1b50fd7af454013a27a287ea333b61d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/add5282f8aec499d80c2b04b2c838217, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2dd5b82e77b44e5fb3aabcdfa45d888f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=138.7 K 2024-11-25T10:56:52,970 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:52,970 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/8609d2d46db04c4289f72ee490b4c707, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a1b50fd7af454013a27a287ea333b61d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/add5282f8aec499d80c2b04b2c838217, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2dd5b82e77b44e5fb3aabcdfa45d888f] 2024-11-25T10:56:52,970 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 79d9522623d641ed90594b10dd06a5f7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732532208689 2024-11-25T10:56:52,971 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8609d2d46db04c4289f72ee490b4c707, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732532208689 2024-11-25T10:56:52,971 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d03d634d07d4fbc8c928db169bc2274, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732532208830 2024-11-25T10:56:52,971 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1b50fd7af454013a27a287ea333b61d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732532208830 2024-11-25T10:56:52,971 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c95f2df945476f99b11f762fd78062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532210156 2024-11-25T10:56:52,972 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bedcf61e19b4d61ac09aa225cb2ee77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732532211461 2024-11-25T10:56:52,972 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting add5282f8aec499d80c2b04b2c838217, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532210156 2024-11-25T10:56:52,972 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dd5b82e77b44e5fb3aabcdfa45d888f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732532211461 2024-11-25T10:56:52,988 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:52,989 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/c24620cc7b1840e0af8375cd5fa06e34 is 50, key is test_row_0/C:col10/1732532211461/Put/seqid=0 2024-11-25T10:56:53,001 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:53,013 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251be35f6635db47b7b0d636bfada7ea02_5dec7eb6196e9e7152a65489b5594c0b store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:53,016 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251be35f6635db47b7b0d636bfada7ea02_5dec7eb6196e9e7152a65489b5594c0b, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:53,016 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251be35f6635db47b7b0d636bfada7ea02_5dec7eb6196e9e7152a65489b5594c0b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:53,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742179_1355 (size=4469) 2024-11-25T10:56:53,063 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#A#compaction#298 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:53,063 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/70a8d74944074e80a1415a5b112ebcb6 is 175, key is test_row_0/A:col10/1732532211461/Put/seqid=0 2024-11-25T10:56:53,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742178_1354 (size=12595) 2024-11-25T10:56:53,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742180_1356 (size=31549) 2024-11-25T10:56:53,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T10:56:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:53,104 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:56:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:53,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259ae2748e3c1240f792013417b41e879b_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532211892/Put/seqid=0 2024-11-25T10:56:53,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:53,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742181_1357 (size=12304) 2024-11-25T10:56:53,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:53,183 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259ae2748e3c1240f792013417b41e879b_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259ae2748e3c1240f792013417b41e879b_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0b56a269b6834cb3824e733cf2d1ea28, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:53,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0b56a269b6834cb3824e733cf2d1ea28 is 175, key is test_row_0/A:col10/1732532211892/Put/seqid=0 2024-11-25T10:56:53,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742182_1358 (size=31105) 2024-11-25T10:56:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T10:56:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532273247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532273252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532273258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532273261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532273383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532273383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532273394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532273383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,496 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/70a8d74944074e80a1415a5b112ebcb6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/70a8d74944074e80a1415a5b112ebcb6 2024-11-25T10:56:53,496 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/c24620cc7b1840e0af8375cd5fa06e34 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c24620cc7b1840e0af8375cd5fa06e34 2024-11-25T10:56:53,504 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/C of 5dec7eb6196e9e7152a65489b5594c0b into c24620cc7b1840e0af8375cd5fa06e34(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:53,504 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:53,504 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/C, priority=12, startTime=1732532212961; duration=0sec 2024-11-25T10:56:53,504 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:53,504 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:C 2024-11-25T10:56:53,504 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:56:53,512 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:56:53,512 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/B is initiating minor compaction (all files) 2024-11-25T10:56:53,512 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/B in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:53,512 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/828cdc9912a04de0ab1a585e6cfdf4f6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/12b6658fcf8346529ccd417a6d673ceb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/0ca08961cdf94138adb3d22fce78ad5a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7c4a562540e641219a462469b36459d7] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=47.8 K 2024-11-25T10:56:53,513 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/A of 5dec7eb6196e9e7152a65489b5594c0b into 70a8d74944074e80a1415a5b112ebcb6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:53,513 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:53,513 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/A, priority=12, startTime=1732532212961; duration=0sec 2024-11-25T10:56:53,513 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:53,513 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:A 2024-11-25T10:56:53,516 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 828cdc9912a04de0ab1a585e6cfdf4f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732532208689 2024-11-25T10:56:53,517 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b6658fcf8346529ccd417a6d673ceb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732532208830 2024-11-25T10:56:53,520 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca08961cdf94138adb3d22fce78ad5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732532210156 2024-11-25T10:56:53,524 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c4a562540e641219a462469b36459d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732532211461 2024-11-25T10:56:53,552 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#B#compaction#300 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:53,552 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/343d2a3c08da410990482fed24b02677 is 50, key is test_row_0/B:col10/1732532211461/Put/seqid=0 2024-11-25T10:56:53,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742183_1359 (size=12595) 2024-11-25T10:56:53,617 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0b56a269b6834cb3824e733cf2d1ea28 2024-11-25T10:56:53,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532273615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bd783befa362408aa98ff3f3f693b15f is 50, key is test_row_0/B:col10/1732532211892/Put/seqid=0 2024-11-25T10:56:53,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532273624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532273625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532273625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742184_1360 (size=12151) 2024-11-25T10:56:53,688 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bd783befa362408aa98ff3f3f693b15f 2024-11-25T10:56:53,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/125c06bd863a4f3e96a6882394533df3 is 50, key is test_row_0/C:col10/1732532211892/Put/seqid=0 2024-11-25T10:56:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T10:56:53,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742185_1361 (size=12151) 2024-11-25T10:56:53,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532273805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,815 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in 5dec7eb6196e9e7152a65489b5594c0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2310 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532273937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532273954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532273954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:53,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532273963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:53,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:53,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:54,018 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/343d2a3c08da410990482fed24b02677 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/343d2a3c08da410990482fed24b02677 2024-11-25T10:56:54,028 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/B of 5dec7eb6196e9e7152a65489b5594c0b into 343d2a3c08da410990482fed24b02677(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:54,028 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:54,028 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/B, priority=12, startTime=1732532212961; duration=0sec 2024-11-25T10:56:54,028 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:54,028 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:B 2024-11-25T10:56:54,173 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/125c06bd863a4f3e96a6882394533df3 2024-11-25T10:56:54,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0b56a269b6834cb3824e733cf2d1ea28 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0b56a269b6834cb3824e733cf2d1ea28 2024-11-25T10:56:54,219 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0b56a269b6834cb3824e733cf2d1ea28, entries=150, sequenceid=204, filesize=30.4 K 2024-11-25T10:56:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bd783befa362408aa98ff3f3f693b15f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bd783befa362408aa98ff3f3f693b15f 2024-11-25T10:56:54,228 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bd783befa362408aa98ff3f3f693b15f, entries=150, sequenceid=204, filesize=11.9 K 2024-11-25T10:56:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-25T10:56:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/125c06bd863a4f3e96a6882394533df3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/125c06bd863a4f3e96a6882394533df3 2024-11-25T10:56:54,284 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/125c06bd863a4f3e96a6882394533df3, entries=150, sequenceid=204, filesize=11.9 K 2024-11-25T10:56:54,293 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 5dec7eb6196e9e7152a65489b5594c0b in 1188ms, sequenceid=204, compaction requested=false 2024-11-25T10:56:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-25T10:56:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-25T10:56:54,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-25T10:56:54,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6670 sec 2024-11-25T10:56:54,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6970 sec 2024-11-25T10:56:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:54,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532213241, max timestamp=1732532214477Num uniques 859; 2024-11-25T10:56:54,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:54,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532213241, max timestamp=1732532214477Num uniques 859; 2024-11-25T10:56:54,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:54,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532213241, max timestamp=1732532214477Num uniques 859; 2024-11-25T10:56:54,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:54,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=40.25 KB heapSize=101.39 KB 2024-11-25T10:56:54,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:54,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:54,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:54,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d874993a045f49e4bd51483b1e16b707_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532214471/Put/seqid=0 2024-11-25T10:56:54,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742186_1362 (size=12304) 2024-11-25T10:56:54,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:54,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d874993a045f49e4bd51483b1e16b707_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d874993a045f49e4bd51483b1e16b707_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:54,564 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a17c1e1e5f6a4041bc3abd1f6c365bbe, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:54,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a17c1e1e5f6a4041bc3abd1f6c365bbe is 175, key is test_row_0/A:col10/1732532214471/Put/seqid=0 2024-11-25T10:56:54,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742187_1363 (size=31105) 2024-11-25T10:56:54,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532214557, max timestamp=1732532214573Num uniques 859; 2024-11-25T10:56:54,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:54,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532214557, max timestamp=1732532214573Num uniques 859; 2024-11-25T10:56:54,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:54,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a17c1e1e5f6a4041bc3abd1f6c365bbe 2024-11-25T10:56:54,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532214557, max timestamp=1732532214573Num uniques 859; 2024-11-25T10:56:54,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:56:54,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/7b23a44da4a54a5a9fb425f2c49729f3 is 50, key is test_row_0/B:col10/1732532214471/Put/seqid=0 2024-11-25T10:56:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742188_1364 (size=12151) 2024-11-25T10:56:54,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732532214629, max timestamp=1732532214651Num uniques 1018; 2024-11-25T10:56:54,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-25T10:56:54,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732532214629, max timestamp=1732532214651Num uniques 1018; 2024-11-25T10:56:54,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-25T10:56:54,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732532214629, max timestamp=1732532214651Num uniques 1018; 2024-11-25T10:56:54,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-25T10:56:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532274736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:54,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532274737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:54,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732532214733, max timestamp=1732532214755Num uniques 1127; 2024-11-25T10:56:54,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-25T10:56:54,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732532214733, max timestamp=1732532214755Num uniques 1127; 2024-11-25T10:56:54,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-25T10:56:54,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732532214733, max timestamp=1732532214755Num uniques 1127; 2024-11-25T10:56:54,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-25T10:56:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T10:56:54,773 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:54,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-25T10:56:54,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T10:56:54,776 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:54,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T10:56:54,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732532214892, max timestamp=1732532214915Num uniques 1086; 2024-11-25T10:56:54,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-25T10:56:54,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732532214892, max timestamp=1732532214915Num uniques 1086; 2024-11-25T10:56:54,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-25T10:56:54,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732532214892, max timestamp=1732532214915Num uniques 1086; 2024-11-25T10:56:54,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-25T10:56:54,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T10:56:54,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:54,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:54,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:54,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532275020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/7b23a44da4a54a5a9fb425f2c49729f3 2024-11-25T10:56:55,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732532215001, max timestamp=1732532215040Num uniques 1095; 2024-11-25T10:56:55,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-25T10:56:55,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732532215001, max timestamp=1732532215040Num uniques 1095; 2024-11-25T10:56:55,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-25T10:56:55,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732532215001, max timestamp=1732532215040Num uniques 1095; 2024-11-25T10:56:55,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-25T10:56:55,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/d7b575d757bc474982f5aa24d615c24f is 50, key is test_row_0/C:col10/1732532214471/Put/seqid=0 2024-11-25T10:56:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T10:56:55,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T10:56:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742189_1365 (size=12151) 2024-11-25T10:56:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532275168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532275172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532275181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532275188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T10:56:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532275297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532275309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532275315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532275325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T10:56:55,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T10:56:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/d7b575d757bc474982f5aa24d615c24f 2024-11-25T10:56:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532275517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532275535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532275549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532275549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/a17c1e1e5f6a4041bc3abd1f6c365bbe as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a17c1e1e5f6a4041bc3abd1f6c365bbe 2024-11-25T10:56:55,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a17c1e1e5f6a4041bc3abd1f6c365bbe, entries=150, sequenceid=229, filesize=30.4 K 2024-11-25T10:56:55,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T10:56:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:55,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/7b23a44da4a54a5a9fb425f2c49729f3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7b23a44da4a54a5a9fb425f2c49729f3 2024-11-25T10:56:55,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7b23a44da4a54a5a9fb425f2c49729f3, entries=150, sequenceid=229, filesize=11.9 K 2024-11-25T10:56:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/d7b575d757bc474982f5aa24d615c24f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/d7b575d757bc474982f5aa24d615c24f 2024-11-25T10:56:55,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/d7b575d757bc474982f5aa24d615c24f, entries=150, sequenceid=229, filesize=11.9 K 2024-11-25T10:56:55,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~101.34 KB/103776, currentSize=147.60 KB/151140 for 5dec7eb6196e9e7152a65489b5594c0b in 1178ms, sequenceid=229, compaction requested=true 2024-11-25T10:56:55,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:55,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:55,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:55,666 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:55,666 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:55,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:55,682 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:55,682 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/B is initiating minor compaction (all files) 2024-11-25T10:56:55,682 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/B in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,682 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/343d2a3c08da410990482fed24b02677, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bd783befa362408aa98ff3f3f693b15f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7b23a44da4a54a5a9fb425f2c49729f3] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=36.0 K 2024-11-25T10:56:55,682 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:55,682 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/A is initiating minor compaction (all files) 2024-11-25T10:56:55,682 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/A in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,682 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/70a8d74944074e80a1415a5b112ebcb6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0b56a269b6834cb3824e733cf2d1ea28, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a17c1e1e5f6a4041bc3abd1f6c365bbe] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=91.6 K 2024-11-25T10:56:55,682 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,682 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/70a8d74944074e80a1415a5b112ebcb6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0b56a269b6834cb3824e733cf2d1ea28, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a17c1e1e5f6a4041bc3abd1f6c365bbe] 2024-11-25T10:56:55,683 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 343d2a3c08da410990482fed24b02677, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732532211461 2024-11-25T10:56:55,693 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70a8d74944074e80a1415a5b112ebcb6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732532211461 2024-11-25T10:56:55,693 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bd783befa362408aa98ff3f3f693b15f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732532211892 2024-11-25T10:56:55,701 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b56a269b6834cb3824e733cf2d1ea28, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732532211892 2024-11-25T10:56:55,701 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b23a44da4a54a5a9fb425f2c49729f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732532214471 2024-11-25T10:56:55,704 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17c1e1e5f6a4041bc3abd1f6c365bbe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732532214471 2024-11-25T10:56:55,723 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:55,723 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/f8f3bae05d8e4cbe85d5f6a74a3ee5e5 is 50, key is test_row_0/B:col10/1732532214471/Put/seqid=0 2024-11-25T10:56:55,741 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:55,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T10:56:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,750 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=147.60 KB heapSize=412.08 KB 2024-11-25T10:56:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:55,757 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125b98e6bfd316c4b9d9d15b7349ca459ae_5dec7eb6196e9e7152a65489b5594c0b store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:55,759 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125b98e6bfd316c4b9d9d15b7349ca459ae_5dec7eb6196e9e7152a65489b5594c0b, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:55,759 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b98e6bfd316c4b9d9d15b7349ca459ae_5dec7eb6196e9e7152a65489b5594c0b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:55,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742190_1366 (size=12697) 2024-11-25T10:56:55,812 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/f8f3bae05d8e4cbe85d5f6a74a3ee5e5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f8f3bae05d8e4cbe85d5f6a74a3ee5e5 2024-11-25T10:56:55,818 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/B of 5dec7eb6196e9e7152a65489b5594c0b into f8f3bae05d8e4cbe85d5f6a74a3ee5e5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:55,818 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:55,819 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/B, priority=13, startTime=1732532215666; duration=0sec 2024-11-25T10:56:55,819 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:55,819 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:B 2024-11-25T10:56:55,822 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:55,825 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:55,825 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/C is initiating minor compaction (all files) 2024-11-25T10:56:55,825 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/C in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:55,825 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c24620cc7b1840e0af8375cd5fa06e34, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/125c06bd863a4f3e96a6882394533df3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/d7b575d757bc474982f5aa24d615c24f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=36.0 K 2024-11-25T10:56:55,826 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c24620cc7b1840e0af8375cd5fa06e34, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732532211461 2024-11-25T10:56:55,826 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 125c06bd863a4f3e96a6882394533df3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732532211892 2024-11-25T10:56:55,827 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d7b575d757bc474982f5aa24d615c24f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732532214471 2024-11-25T10:56:55,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742191_1367 (size=4469) 2024-11-25T10:56:55,833 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#A#compaction#307 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:55,834 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/2eb8a05ab8604961b922ad45d86872c2 is 175, key is test_row_0/A:col10/1732532214471/Put/seqid=0 2024-11-25T10:56:55,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112552e3ca683dbd43ce84e6557177b0ec5a_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532215132/Put/seqid=0 2024-11-25T10:56:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:55,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532275864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,890 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:55,890 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/bcab3784ac814bd4b8c297ea76afc0cb is 50, key is test_row_0/C:col10/1732532214471/Put/seqid=0 2024-11-25T10:56:55,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532275875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532275883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532275885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742192_1368 (size=31651) 2024-11-25T10:56:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742193_1369 (size=12454) 2024-11-25T10:56:55,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:55,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532275887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T10:56:55,918 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112552e3ca683dbd43ce84e6557177b0ec5a_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112552e3ca683dbd43ce84e6557177b0ec5a_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:55,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/30c5cb08b9c046df9841589978aa7e9f, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:55,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/30c5cb08b9c046df9841589978aa7e9f is 175, key is test_row_0/A:col10/1732532215132/Put/seqid=0 2024-11-25T10:56:55,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742194_1370 (size=12697) 2024-11-25T10:56:55,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742195_1371 (size=31255) 2024-11-25T10:56:55,983 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/bcab3784ac814bd4b8c297ea76afc0cb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/bcab3784ac814bd4b8c297ea76afc0cb 2024-11-25T10:56:55,995 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/C of 5dec7eb6196e9e7152a65489b5594c0b into bcab3784ac814bd4b8c297ea76afc0cb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:55,995 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:55,995 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/C, priority=13, startTime=1732532215667; duration=0sec 2024-11-25T10:56:55,995 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:55,995 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:C 2024-11-25T10:56:56,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532276004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532276004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532276004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532276017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532276224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532276224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532276224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532276244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,309 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/2eb8a05ab8604961b922ad45d86872c2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2eb8a05ab8604961b922ad45d86872c2 2024-11-25T10:56:56,319 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/A of 5dec7eb6196e9e7152a65489b5594c0b into 2eb8a05ab8604961b922ad45d86872c2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:56,319 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:56,319 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/A, priority=13, startTime=1732532215666; duration=0sec 2024-11-25T10:56:56,319 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:56,319 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:A 2024-11-25T10:56:56,394 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/30c5cb08b9c046df9841589978aa7e9f 2024-11-25T10:56:56,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532276397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bf18c40235bc4dacbbab1998922d4359 is 50, key is test_row_0/B:col10/1732532215132/Put/seqid=0 2024-11-25T10:56:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742196_1372 (size=12301) 2024-11-25T10:56:56,448 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bf18c40235bc4dacbbab1998922d4359 2024-11-25T10:56:56,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/ea3c685af0b549f4b5c37141a8642f3d is 50, key is test_row_0/C:col10/1732532215132/Put/seqid=0 2024-11-25T10:56:56,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742197_1373 (size=12301) 2024-11-25T10:56:56,541 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/ea3c685af0b549f4b5c37141a8642f3d 2024-11-25T10:56:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/30c5cb08b9c046df9841589978aa7e9f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/30c5cb08b9c046df9841589978aa7e9f 2024-11-25T10:56:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532276554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532276554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532276554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,580 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/30c5cb08b9c046df9841589978aa7e9f, entries=150, sequenceid=334, filesize=30.5 K 2024-11-25T10:56:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:56,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/bf18c40235bc4dacbbab1998922d4359 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf18c40235bc4dacbbab1998922d4359 2024-11-25T10:56:56,593 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf18c40235bc4dacbbab1998922d4359, entries=150, sequenceid=334, filesize=12.0 K 2024-11-25T10:56:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/ea3c685af0b549f4b5c37141a8642f3d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/ea3c685af0b549f4b5c37141a8642f3d 2024-11-25T10:56:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,606 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/ea3c685af0b549f4b5c37141a8642f3d, entries=150, sequenceid=334, filesize=12.0 K 2024-11-25T10:56:56,608 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~412.03 KB/421920, currentSize=33.54 KB/34350 for 5dec7eb6196e9e7152a65489b5594c0b in 857ms, sequenceid=334, compaction requested=false 2024-11-25T10:56:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-25T10:56:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-25T10:56:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-25T10:56:56,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8420 sec 2024-11-25T10:56:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8510 sec 2024-11-25T10:56:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:56:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e3b2a40f76ec4cb18e59bf3667fcf038_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532216734/Put/seqid=0 2024-11-25T10:56:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742198_1374 (size=12454) 2024-11-25T10:56:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,797 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,815 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e3b2a40f76ec4cb18e59bf3667fcf038_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e3b2a40f76ec4cb18e59bf3667fcf038_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,821 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e702d9f89c2c497aa82dabbaf2ac9019, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e702d9f89c2c497aa82dabbaf2ac9019 is 175, key is test_row_0/A:col10/1732532216734/Put/seqid=0 2024-11-25T10:56:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742199_1375 (size=31251) 2024-11-25T10:56:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,851 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e702d9f89c2c497aa82dabbaf2ac9019 2024-11-25T10:56:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/d73362c55a1848cfadae93aea6732b32 is 50, key is test_row_0/B:col10/1732532216734/Put/seqid=0 2024-11-25T10:56:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T10:56:56,934 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:56:56,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:56:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-25T10:56:56,937 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:56:56,939 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:56:56,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:56:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T10:56:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742200_1376 (size=9857) 2024-11-25T10:56:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/d73362c55a1848cfadae93aea6732b32 2024-11-25T10:56:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:56,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/93ceaff4be6e49b5bfe274b2fd9c8a2e is 50, key is test_row_0/C:col10/1732532216734/Put/seqid=0 2024-11-25T10:56:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T10:56:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742201_1377 (size=9857) 2024-11-25T10:56:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/93ceaff4be6e49b5bfe274b2fd9c8a2e 2024-11-25T10:56:57,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/e702d9f89c2c497aa82dabbaf2ac9019 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e702d9f89c2c497aa82dabbaf2ac9019 2024-11-25T10:56:57,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e702d9f89c2c497aa82dabbaf2ac9019, entries=150, sequenceid=347, filesize=30.5 K 2024-11-25T10:56:57,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/d73362c55a1848cfadae93aea6732b32 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/d73362c55a1848cfadae93aea6732b32 2024-11-25T10:56:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532217140, max timestamp=1732532217170Num uniques 859; 2024-11-25T10:56:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532217140, max timestamp=1732532217170Num uniques 859; 2024-11-25T10:56:57,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:57,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532217140, max timestamp=1732532217170Num uniques 859; 2024-11-25T10:56:57,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:56:57,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/d73362c55a1848cfadae93aea6732b32, entries=100, sequenceid=347, filesize=9.6 K 2024-11-25T10:56:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/93ceaff4be6e49b5bfe274b2fd9c8a2e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/93ceaff4be6e49b5bfe274b2fd9c8a2e 2024-11-25T10:56:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/93ceaff4be6e49b5bfe274b2fd9c8a2e, entries=100, sequenceid=347, filesize=9.6 K 2024-11-25T10:56:57,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=46.96 KB/48090 for 5dec7eb6196e9e7152a65489b5594c0b in 475ms, sequenceid=347, compaction requested=true 2024-11-25T10:56:57,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:57,211 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:56:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:57,212 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:56:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:56:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:57,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=53.67 KB heapSize=141.23 KB 2024-11-25T10:56:57,217 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94157 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:57,217 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/A is initiating minor compaction (all files) 2024-11-25T10:56:57,217 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/A in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,217 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2eb8a05ab8604961b922ad45d86872c2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/30c5cb08b9c046df9841589978aa7e9f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e702d9f89c2c497aa82dabbaf2ac9019] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=92.0 K 2024-11-25T10:56:57,217 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,217 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2eb8a05ab8604961b922ad45d86872c2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/30c5cb08b9c046df9841589978aa7e9f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e702d9f89c2c497aa82dabbaf2ac9019] 2024-11-25T10:56:57,218 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb8a05ab8604961b922ad45d86872c2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732532214471 2024-11-25T10:56:57,218 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:57,218 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/B is initiating minor compaction (all files) 2024-11-25T10:56:57,218 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/B in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,218 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f8f3bae05d8e4cbe85d5f6a74a3ee5e5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf18c40235bc4dacbbab1998922d4359, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/d73362c55a1848cfadae93aea6732b32] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=34.0 K 2024-11-25T10:56:57,219 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30c5cb08b9c046df9841589978aa7e9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732532215132 2024-11-25T10:56:57,224 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f8f3bae05d8e4cbe85d5f6a74a3ee5e5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732532214471 2024-11-25T10:56:57,225 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e702d9f89c2c497aa82dabbaf2ac9019, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732532215882 2024-11-25T10:56:57,225 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bf18c40235bc4dacbbab1998922d4359, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732532215132 2024-11-25T10:56:57,226 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d73362c55a1848cfadae93aea6732b32, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732532215882 2024-11-25T10:56:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:57,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T10:56:57,257 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:57,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:57,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:57,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bce3b9086a994b6294111eb3efee2bf9_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532217175/Put/seqid=0 2024-11-25T10:56:57,275 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:57,276 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/108bbcab9bf848f3bc50e6bbc8218ab2 is 50, key is test_row_0/B:col10/1732532216734/Put/seqid=0 2024-11-25T10:56:57,297 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411256eaf9ab8460a449cba992b84e32745be_5dec7eb6196e9e7152a65489b5594c0b store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:57,299 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411256eaf9ab8460a449cba992b84e32745be_5dec7eb6196e9e7152a65489b5594c0b, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:57,299 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256eaf9ab8460a449cba992b84e32745be_5dec7eb6196e9e7152a65489b5594c0b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:57,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742203_1379 (size=12949) 2024-11-25T10:56:57,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532277313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532277314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742204_1380 (size=4469) 2024-11-25T10:56:57,352 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#A#compaction#315 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:57,353 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/ee3258b6216a4e4797649366ea777d04 is 175, key is test_row_0/A:col10/1732532216734/Put/seqid=0 2024-11-25T10:56:57,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742202_1378 (size=17534) 2024-11-25T10:56:57,354 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/108bbcab9bf848f3bc50e6bbc8218ab2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/108bbcab9bf848f3bc50e6bbc8218ab2 2024-11-25T10:56:57,356 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532277337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532277336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,370 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/B of 5dec7eb6196e9e7152a65489b5594c0b into 108bbcab9bf848f3bc50e6bbc8218ab2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:57,370 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:57,370 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/B, priority=13, startTime=1732532217212; duration=0sec 2024-11-25T10:56:57,370 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:56:57,370 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:B 2024-11-25T10:56:57,370 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:56:57,373 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bce3b9086a994b6294111eb3efee2bf9_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bce3b9086a994b6294111eb3efee2bf9_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:57,378 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:56:57,378 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/C is initiating minor compaction (all files) 2024-11-25T10:56:57,378 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/C in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,378 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/bcab3784ac814bd4b8c297ea76afc0cb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/ea3c685af0b549f4b5c37141a8642f3d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/93ceaff4be6e49b5bfe274b2fd9c8a2e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=34.0 K 2024-11-25T10:56:57,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/be6d135065804d6bac060675b5a39a02, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:57,378 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bcab3784ac814bd4b8c297ea76afc0cb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732532214471 2024-11-25T10:56:57,379 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3c685af0b549f4b5c37141a8642f3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732532215132 2024-11-25T10:56:57,380 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ceaff4be6e49b5bfe274b2fd9c8a2e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732532215882 2024-11-25T10:56:57,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/be6d135065804d6bac060675b5a39a02 is 175, key is test_row_0/A:col10/1732532217175/Put/seqid=0 2024-11-25T10:56:57,391 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:56:57,391 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/e82791440abe4e57aa4283cce07645aa is 50, key is test_row_0/C:col10/1732532216734/Put/seqid=0 2024-11-25T10:56:57,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742205_1381 (size=32010) 2024-11-25T10:56:57,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742206_1382 (size=48639) 2024-11-25T10:56:57,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742207_1383 (size=12949) 2024-11-25T10:56:57,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532277424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,437 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_1' in 5dec7eb6196e9e7152a65489b5594c0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532277450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,472 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/ee3258b6216a4e4797649366ea777d04 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee3258b6216a4e4797649366ea777d04 2024-11-25T10:56:57,482 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/A of 5dec7eb6196e9e7152a65489b5594c0b into ee3258b6216a4e4797649366ea777d04(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:57,482 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:57,482 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/A, priority=13, startTime=1732532217211; duration=0sec 2024-11-25T10:56:57,482 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:57,482 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:A 2024-11-25T10:56:57,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532277458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532277463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532277477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T10:56:57,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532277677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532277694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532277694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532277694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:57,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:57,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/be6d135065804d6bac060675b5a39a02 2024-11-25T10:56:57,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/2b47aad043c14b46893c5aa2f9cc0934 is 50, key is test_row_0/B:col10/1732532217175/Put/seqid=0 2024-11-25T10:56:57,876 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/e82791440abe4e57aa4283cce07645aa as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/e82791440abe4e57aa4283cce07645aa 2024-11-25T10:56:57,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:57,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:57,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742208_1384 (size=12301) 2024-11-25T10:56:57,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/2b47aad043c14b46893c5aa2f9cc0934 2024-11-25T10:56:57,920 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/C of 5dec7eb6196e9e7152a65489b5594c0b into e82791440abe4e57aa4283cce07645aa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:56:57,920 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:57,920 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/C, priority=13, startTime=1732532217212; duration=0sec 2024-11-25T10:56:57,920 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:56:57,920 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:C 2024-11-25T10:56:57,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/cc182813536e42df87279e48671fca30 is 50, key is test_row_0/C:col10/1732532217175/Put/seqid=0 2024-11-25T10:56:57,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742209_1385 (size=12301) 2024-11-25T10:56:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532278005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532278005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532278014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532278017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:58,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T10:56:58,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:58,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:58,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:58,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:58,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:58,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:58,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:56:58,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/cc182813536e42df87279e48671fca30 2024-11-25T10:56:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/be6d135065804d6bac060675b5a39a02 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/be6d135065804d6bac060675b5a39a02 2024-11-25T10:56:58,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/be6d135065804d6bac060675b5a39a02, entries=250, sequenceid=373, filesize=47.5 K 2024-11-25T10:56:58,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/2b47aad043c14b46893c5aa2f9cc0934 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/2b47aad043c14b46893c5aa2f9cc0934 2024-11-25T10:56:58,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/2b47aad043c14b46893c5aa2f9cc0934, entries=150, sequenceid=373, filesize=12.0 K 2024-11-25T10:56:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/cc182813536e42df87279e48671fca30 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/cc182813536e42df87279e48671fca30 2024-11-25T10:56:58,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/cc182813536e42df87279e48671fca30, entries=150, sequenceid=373, filesize=12.0 K 2024-11-25T10:56:58,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~181.03 KB/185376, currentSize=114.05 KB/116790 for 5dec7eb6196e9e7152a65489b5594c0b in 1240ms, sequenceid=373, compaction requested=false 2024-11-25T10:56:58,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:58,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T10:56:58,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:58,533 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=114.05 KB heapSize=339.41 KB 2024-11-25T10:56:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:56:58,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:56:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a67bab0d1b0d4deb9c0072aa97923041_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532217293/Put/seqid=0 2024-11-25T10:56:58,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532278565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532278571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742210_1386 (size=14994) 2024-11-25T10:56:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:56:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532278577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532278585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,609 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a67bab0d1b0d4deb9c0072aa97923041_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a67bab0d1b0d4deb9c0072aa97923041_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:58,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/9135cabebabe4dc3b977cbe58474d7dd, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:56:58,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/9135cabebabe4dc3b977cbe58474d7dd is 175, key is test_row_0/A:col10/1732532217293/Put/seqid=0 2024-11-25T10:56:58,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742211_1387 (size=39949) 2024-11-25T10:56:58,656 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/9135cabebabe4dc3b977cbe58474d7dd 2024-11-25T10:56:58,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/4a1a0b58b43b40e1bb2d3512e76b730d is 50, key is test_row_0/B:col10/1732532217293/Put/seqid=0 2024-11-25T10:56:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532278693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532278693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742212_1388 (size=12301) 2024-11-25T10:56:58,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532278717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532278720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532278924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532278933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532278944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532278948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T10:56:59,131 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/4a1a0b58b43b40e1bb2d3512e76b730d 2024-11-25T10:56:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f7b59e4b65d7477a863716e8b35959f1 is 50, key is test_row_0/C:col10/1732532217293/Put/seqid=0 2024-11-25T10:56:59,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742213_1389 (size=12301) 2024-11-25T10:56:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532279243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532279254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532279264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532279264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36872 deadline: 1732532279455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,466 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_1' in 5dec7eb6196e9e7152a65489b5594c0b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:56:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,595 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f7b59e4b65d7477a863716e8b35959f1 2024-11-25T10:56:59,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/9135cabebabe4dc3b977cbe58474d7dd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/9135cabebabe4dc3b977cbe58474d7dd 2024-11-25T10:56:59,613 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/9135cabebabe4dc3b977cbe58474d7dd, entries=200, sequenceid=397, filesize=39.0 K 2024-11-25T10:56:59,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/4a1a0b58b43b40e1bb2d3512e76b730d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4a1a0b58b43b40e1bb2d3512e76b730d 2024-11-25T10:56:59,627 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4a1a0b58b43b40e1bb2d3512e76b730d, entries=150, sequenceid=397, filesize=12.0 K 2024-11-25T10:56:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/f7b59e4b65d7477a863716e8b35959f1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f7b59e4b65d7477a863716e8b35959f1 2024-11-25T10:56:59,649 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f7b59e4b65d7477a863716e8b35959f1, entries=150, sequenceid=397, filesize=12.0 K 2024-11-25T10:56:59,650 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 5dec7eb6196e9e7152a65489b5594c0b in 1117ms, sequenceid=397, compaction requested=true 2024-11-25T10:56:59,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:56:59,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:56:59,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-25T10:56:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-25T10:56:59,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-25T10:56:59,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7250 sec 2024-11-25T10:56:59,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.7320 sec 2024-11-25T10:56:59,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T10:56:59,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:56:59,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:59,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:56:59,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:59,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:56:59,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:56:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:56:59,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bab3fb6506e94baebadd558dcb1c2b3b_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532219777/Put/seqid=0 2024-11-25T10:56:59,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742214_1390 (size=14994) 2024-11-25T10:56:59,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532279913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532279915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532279919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:56:59,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:56:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532279921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:56:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:56:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532280053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532280053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532280054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532280057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,253 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:00,287 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bab3fb6506e94baebadd558dcb1c2b3b_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bab3fb6506e94baebadd558dcb1c2b3b_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:00,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:57:00,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d is 175, key is test_row_0/A:col10/1732532219777/Put/seqid=0 2024-11-25T10:57:00,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532280284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532280284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532280286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532280288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742215_1391 (size=39949) 2024-11-25T10:57:00,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532280614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532280614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532280615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532280617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:00,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d 2024-11-25T10:57:00,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/112af6664cf0478c88c70c0c42bcc6d3 is 50, key is test_row_0/B:col10/1732532219777/Put/seqid=0 2024-11-25T10:57:00,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742216_1392 (size=12301) 2024-11-25T10:57:00,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/112af6664cf0478c88c70c0c42bcc6d3 2024-11-25T10:57:00,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9774ae40dec647d488ccd6fff53e9107 is 50, key is test_row_0/C:col10/1732532219777/Put/seqid=0 2024-11-25T10:57:00,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742217_1393 (size=12301) 2024-11-25T10:57:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T10:57:01,104 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:01,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-25T10:57:01,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:01,109 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:01,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T10:57:01,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36888 deadline: 1732532281129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36842 deadline: 1732532281139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:01,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36854 deadline: 1732532281141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:01,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:01,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36858 deadline: 1732532281149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:01,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:01,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T10:57:01,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T10:57:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. as already flushing 2024-11-25T10:57:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:01,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:01,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9774ae40dec647d488ccd6fff53e9107 2024-11-25T10:57:01,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d 2024-11-25T10:57:01,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d, entries=200, sequenceid=409, filesize=39.0 K 2024-11-25T10:57:01,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/112af6664cf0478c88c70c0c42bcc6d3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/112af6664cf0478c88c70c0c42bcc6d3 2024-11-25T10:57:01,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/112af6664cf0478c88c70c0c42bcc6d3, entries=150, sequenceid=409, filesize=12.0 K 2024-11-25T10:57:01,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/9774ae40dec647d488ccd6fff53e9107 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9774ae40dec647d488ccd6fff53e9107 2024-11-25T10:57:01,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9774ae40dec647d488ccd6fff53e9107, entries=150, sequenceid=409, filesize=12.0 K 2024-11-25T10:57:01,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5dec7eb6196e9e7152a65489b5594c0b in 1611ms, sequenceid=409, compaction requested=true 2024-11-25T10:57:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:57:01,390 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:01,392 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160547 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:01,392 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/A is initiating minor compaction (all files) 2024-11-25T10:57:01,392 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/A in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,392 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee3258b6216a4e4797649366ea777d04, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/be6d135065804d6bac060675b5a39a02, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/9135cabebabe4dc3b977cbe58474d7dd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=156.8 K 2024-11-25T10:57:01,392 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,392 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee3258b6216a4e4797649366ea777d04, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/be6d135065804d6bac060675b5a39a02, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/9135cabebabe4dc3b977cbe58474d7dd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d] 2024-11-25T10:57:01,393 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3258b6216a4e4797649366ea777d04, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732532215185 2024-11-25T10:57:01,393 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting be6d135065804d6bac060675b5a39a02, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732532217166 2024-11-25T10:57:01,393 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9135cabebabe4dc3b977cbe58474d7dd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732532217293 2024-11-25T10:57:01,393 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8bd65c7eaa487a8c4d5b0828ca0b2d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732532218565 2024-11-25T10:57:01,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:01,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:01,401 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:01,401 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:01,401 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/B is initiating minor compaction (all files) 2024-11-25T10:57:01,402 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/B in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,402 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/108bbcab9bf848f3bc50e6bbc8218ab2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/2b47aad043c14b46893c5aa2f9cc0934, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4a1a0b58b43b40e1bb2d3512e76b730d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/112af6664cf0478c88c70c0c42bcc6d3] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=48.7 K 2024-11-25T10:57:01,402 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 108bbcab9bf848f3bc50e6bbc8218ab2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732532215185 2024-11-25T10:57:01,402 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b47aad043c14b46893c5aa2f9cc0934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732532217175 2024-11-25T10:57:01,403 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a1a0b58b43b40e1bb2d3512e76b730d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732532217293 2024-11-25T10:57:01,403 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 112af6664cf0478c88c70c0c42bcc6d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732532218574 2024-11-25T10:57:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5dec7eb6196e9e7152a65489b5594c0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:01,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T10:57:01,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,418 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T10:57:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:57:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:57:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:57:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:01,421 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:57:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T10:57:01,424 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:01,425 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/19242dfb183d439f9c8e4abb02398f15 is 50, key is test_row_0/B:col10/1732532219777/Put/seqid=0 2024-11-25T10:57:01,426 INFO [Thread-1402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1402. 2024-11-25T10:57:01,426 DEBUG [Thread-1402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:01,426 DEBUG [Thread-1402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,426 DEBUG [Thread-1402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,426 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:01,426 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125c7b7e6f14ae0426699d7a49b087b9a46_5dec7eb6196e9e7152a65489b5594c0b store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:57:01,427 INFO [Thread-1408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1408. 2024-11-25T10:57:01,428 DEBUG [Thread-1408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:01,428 DEBUG [Thread-1408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,428 DEBUG [Thread-1408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:01,429 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125c7b7e6f14ae0426699d7a49b087b9a46_5dec7eb6196e9e7152a65489b5594c0b, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:57:01,430 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c7b7e6f14ae0426699d7a49b087b9a46_5dec7eb6196e9e7152a65489b5594c0b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:57:01,431 INFO [Thread-1410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1410. 2024-11-25T10:57:01,431 DEBUG [Thread-1410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:01,431 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,431 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:01,441 INFO [Thread-1406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1406. 2024-11-25T10:57:01,441 DEBUG [Thread-1406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:01,441 DEBUG [Thread-1406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,441 DEBUG [Thread-1406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:01,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742218_1394 (size=13085) 2024-11-25T10:57:01,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742219_1395 (size=4469) 2024-11-25T10:57:01,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112530116cec2dca40cfb130d7925c3c1447_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532219907/Put/seqid=0 2024-11-25T10:57:01,445 INFO [Thread-1404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1404. 2024-11-25T10:57:01,445 DEBUG [Thread-1404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:01,445 DEBUG [Thread-1404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,445 DEBUG [Thread-1404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:01,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742220_1396 (size=12454) 2024-11-25T10:57:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:01,454 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/19242dfb183d439f9c8e4abb02398f15 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/19242dfb183d439f9c8e4abb02398f15 2024-11-25T10:57:01,457 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112530116cec2dca40cfb130d7925c3c1447_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112530116cec2dca40cfb130d7925c3c1447_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:01,462 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/B of 5dec7eb6196e9e7152a65489b5594c0b into 19242dfb183d439f9c8e4abb02398f15(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:01,462 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:57:01,462 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/B, priority=12, startTime=1732532221400; duration=0sec 2024-11-25T10:57:01,462 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:01,462 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:B 2024-11-25T10:57:01,462 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:01,463 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:01,463 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): 5dec7eb6196e9e7152a65489b5594c0b/C is initiating minor compaction (all files) 2024-11-25T10:57:01,463 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5dec7eb6196e9e7152a65489b5594c0b/C in TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,463 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/e82791440abe4e57aa4283cce07645aa, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/cc182813536e42df87279e48671fca30, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f7b59e4b65d7477a863716e8b35959f1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9774ae40dec647d488ccd6fff53e9107] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp, totalSize=48.7 K 2024-11-25T10:57:01,464 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82791440abe4e57aa4283cce07645aa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732532215185 2024-11-25T10:57:01,464 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc182813536e42df87279e48671fca30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732532217175 2024-11-25T10:57:01,464 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b59e4b65d7477a863716e8b35959f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732532217293 2024-11-25T10:57:01,465 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9774ae40dec647d488ccd6fff53e9107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732532218574 2024-11-25T10:57:01,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/c6277cec16484dbcbcd4513e91e08623, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:57:01,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/c6277cec16484dbcbcd4513e91e08623 is 175, key is test_row_0/A:col10/1732532219907/Put/seqid=0 2024-11-25T10:57:01,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742221_1397 (size=31255) 2024-11-25T10:57:01,473 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/c6277cec16484dbcbcd4513e91e08623 2024-11-25T10:57:01,478 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#C#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:01,479 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/b825d20bf9ea46289f737547ac2d5e05 is 50, key is test_row_0/C:col10/1732532219777/Put/seqid=0 2024-11-25T10:57:01,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/f356b048f4cd483ab21204ee807fef57 is 50, key is test_row_0/B:col10/1732532219907/Put/seqid=0 2024-11-25T10:57:01,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742222_1398 (size=13085) 2024-11-25T10:57:01,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742223_1399 (size=12301) 2024-11-25T10:57:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T10:57:01,845 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5dec7eb6196e9e7152a65489b5594c0b#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:01,845 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/6d91d93f1a9149adb8cd4f5df31c11c6 is 175, key is test_row_0/A:col10/1732532219777/Put/seqid=0 2024-11-25T10:57:01,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742224_1400 (size=32039) 2024-11-25T10:57:01,898 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/b825d20bf9ea46289f737547ac2d5e05 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/b825d20bf9ea46289f737547ac2d5e05 2024-11-25T10:57:01,899 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/f356b048f4cd483ab21204ee807fef57 2024-11-25T10:57:01,903 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/C of 5dec7eb6196e9e7152a65489b5594c0b into b825d20bf9ea46289f737547ac2d5e05(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:01,903 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:57:01,904 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/C, priority=12, startTime=1732532221408; duration=0sec 2024-11-25T10:57:01,904 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:01,904 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:C 2024-11-25T10:57:01,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/31004cd2846a42c3a408c0f47c3c298f is 50, key is test_row_0/C:col10/1732532219907/Put/seqid=0 2024-11-25T10:57:01,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742225_1401 (size=12301) 2024-11-25T10:57:01,917 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/31004cd2846a42c3a408c0f47c3c298f 2024-11-25T10:57:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/c6277cec16484dbcbcd4513e91e08623 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/c6277cec16484dbcbcd4513e91e08623 2024-11-25T10:57:01,931 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/c6277cec16484dbcbcd4513e91e08623, entries=150, sequenceid=429, filesize=30.5 K 2024-11-25T10:57:01,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/f356b048f4cd483ab21204ee807fef57 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f356b048f4cd483ab21204ee807fef57 2024-11-25T10:57:01,936 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f356b048f4cd483ab21204ee807fef57, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T10:57:01,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/31004cd2846a42c3a408c0f47c3c298f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/31004cd2846a42c3a408c0f47c3c298f 2024-11-25T10:57:01,940 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/31004cd2846a42c3a408c0f47c3c298f, entries=150, sequenceid=429, filesize=12.0 K 2024-11-25T10:57:01,941 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 5dec7eb6196e9e7152a65489b5594c0b in 523ms, sequenceid=429, compaction requested=false 2024-11-25T10:57:01,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:57:01,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:01,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-25T10:57:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-25T10:57:01,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-25T10:57:01,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 834 msec 2024-11-25T10:57:01,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 841 msec 2024-11-25T10:57:02,155 INFO [Thread-1391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1391. 2024-11-25T10:57:02,155 DEBUG [Thread-1391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:02,155 DEBUG [Thread-1391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,155 DEBUG [Thread-1391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:02,165 INFO [Thread-1397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1397. 2024-11-25T10:57:02,165 DEBUG [Thread-1397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:02,165 DEBUG [Thread-1397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,166 DEBUG [Thread-1397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:02,174 INFO [Thread-1395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1395. 2024-11-25T10:57:02,174 DEBUG [Thread-1395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:02,174 DEBUG [Thread-1395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,175 DEBUG [Thread-1395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:02,185 INFO [Thread-1393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1393. 2024-11-25T10:57:02,185 DEBUG [Thread-1393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:02,185 DEBUG [Thread-1393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,185 DEBUG [Thread-1393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:02,185 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T10:57:02,244 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:02,255 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/6d91d93f1a9149adb8cd4f5df31c11c6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6d91d93f1a9149adb8cd4f5df31c11c6 2024-11-25T10:57:02,260 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5dec7eb6196e9e7152a65489b5594c0b/A of 5dec7eb6196e9e7152a65489b5594c0b into 6d91d93f1a9149adb8cd4f5df31c11c6(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:02,260 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5dec7eb6196e9e7152a65489b5594c0b: 2024-11-25T10:57:02,260 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., storeName=5dec7eb6196e9e7152a65489b5594c0b/A, priority=12, startTime=1732532221390; duration=0sec 2024-11-25T10:57:02,260 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:02,260 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5dec7eb6196e9e7152a65489b5594c0b:A 2024-11-25T10:57:03,476 INFO [Thread-1399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1399. 2024-11-25T10:57:03,476 DEBUG [Thread-1399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:03,476 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:03,476 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:03,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 561 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1672 rows 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 554 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1654 rows 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 562 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1674 rows 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 557 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1669 rows 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 576 2024-11-25T10:57:03,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1704 rows 2024-11-25T10:57:03,477 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:57:03,477 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:57:03,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:03,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:03,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:03,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T10:57:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T10:57:03,502 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532223501"}]},"ts":"1732532223501"} 2024-11-25T10:57:03,526 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T10:57:03,526 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T10:57:03,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:57:03,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, UNASSIGN}] 2024-11-25T10:57:03,532 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, UNASSIGN 2024-11-25T10:57:03,533 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:03,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, UNASSIGN because future has completed 2024-11-25T10:57:03,536 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:57:03,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:57:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T10:57:03,698 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing 5dec7eb6196e9e7152a65489b5594c0b, disabling compactions & flushes 2024-11-25T10:57:03,699 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. after waiting 0 ms 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:03,699 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing 5dec7eb6196e9e7152a65489b5594c0b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=A 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=B 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:03,699 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5dec7eb6196e9e7152a65489b5594c0b, store=C 2024-11-25T10:57:03,700 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:03,709 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aaa6991b5dcc4c5f9fa5e5e455dc7f0d_5dec7eb6196e9e7152a65489b5594c0b is 50, key is test_row_0/A:col10/1732532222184/Put/seqid=0 2024-11-25T10:57:03,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742226_1402 (size=12454) 2024-11-25T10:57:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T10:57:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T10:57:04,157 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:04,180 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125aaa6991b5dcc4c5f9fa5e5e455dc7f0d_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aaa6991b5dcc4c5f9fa5e5e455dc7f0d_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:04,185 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f48d8541a4e8437b8d447bb4557ff984, store: [table=TestAcidGuarantees family=A region=5dec7eb6196e9e7152a65489b5594c0b] 2024-11-25T10:57:04,186 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f48d8541a4e8437b8d447bb4557ff984 is 175, key is test_row_0/A:col10/1732532222184/Put/seqid=0 2024-11-25T10:57:04,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742227_1403 (size=31255) 2024-11-25T10:57:04,624 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f48d8541a4e8437b8d447bb4557ff984 2024-11-25T10:57:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T10:57:04,642 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/11677151f1c445d7933bed2304e7ac67 is 50, key is test_row_0/B:col10/1732532222184/Put/seqid=0 2024-11-25T10:57:04,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742228_1404 (size=12301) 2024-11-25T10:57:05,077 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/11677151f1c445d7933bed2304e7ac67 2024-11-25T10:57:05,090 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/2e2922efa96f49f298ba6aadc8ae5775 is 50, key is test_row_0/C:col10/1732532222184/Put/seqid=0 2024-11-25T10:57:05,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742229_1405 (size=12301) 2024-11-25T10:57:05,507 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/2e2922efa96f49f298ba6aadc8ae5775 2024-11-25T10:57:05,511 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/A/f48d8541a4e8437b8d447bb4557ff984 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f48d8541a4e8437b8d447bb4557ff984 2024-11-25T10:57:05,516 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f48d8541a4e8437b8d447bb4557ff984, entries=150, sequenceid=440, filesize=30.5 K 2024-11-25T10:57:05,516 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/B/11677151f1c445d7933bed2304e7ac67 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/11677151f1c445d7933bed2304e7ac67 2024-11-25T10:57:05,521 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/11677151f1c445d7933bed2304e7ac67, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T10:57:05,525 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/.tmp/C/2e2922efa96f49f298ba6aadc8ae5775 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2e2922efa96f49f298ba6aadc8ae5775 2024-11-25T10:57:05,528 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2e2922efa96f49f298ba6aadc8ae5775, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T10:57:05,529 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5dec7eb6196e9e7152a65489b5594c0b in 1830ms, sequenceid=440, compaction requested=true 2024-11-25T10:57:05,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f491dbcce1d942f9aa715ec7b42b740c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/51ba17e410f34bfca209e0f7c4cf15d3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee17c694226442c2a7f04ded891b343a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/68b7988eac6a4207b2d414717235030d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/5c3af7ef17634d558e9836a593fda9a7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/17338d38aae646599989ca029909673b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6f86920482bb4272a312dd166013771d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e386e90f693448c2891f229efa67e9ba, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/8609d2d46db04c4289f72ee490b4c707, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f3bb396041264294bce3d1c3e886a0b0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a1b50fd7af454013a27a287ea333b61d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/add5282f8aec499d80c2b04b2c838217, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2dd5b82e77b44e5fb3aabcdfa45d888f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/70a8d74944074e80a1415a5b112ebcb6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0b56a269b6834cb3824e733cf2d1ea28, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2eb8a05ab8604961b922ad45d86872c2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a17c1e1e5f6a4041bc3abd1f6c365bbe, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/30c5cb08b9c046df9841589978aa7e9f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee3258b6216a4e4797649366ea777d04, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e702d9f89c2c497aa82dabbaf2ac9019, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/be6d135065804d6bac060675b5a39a02, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/9135cabebabe4dc3b977cbe58474d7dd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d] to archive 2024-11-25T10:57:05,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:05,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f491dbcce1d942f9aa715ec7b42b740c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f491dbcce1d942f9aa715ec7b42b740c 2024-11-25T10:57:05,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/51ba17e410f34bfca209e0f7c4cf15d3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/51ba17e410f34bfca209e0f7c4cf15d3 2024-11-25T10:57:05,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee17c694226442c2a7f04ded891b343a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee17c694226442c2a7f04ded891b343a 2024-11-25T10:57:05,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/68b7988eac6a4207b2d414717235030d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/68b7988eac6a4207b2d414717235030d 2024-11-25T10:57:05,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/5c3af7ef17634d558e9836a593fda9a7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/5c3af7ef17634d558e9836a593fda9a7 2024-11-25T10:57:05,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/17338d38aae646599989ca029909673b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/17338d38aae646599989ca029909673b 2024-11-25T10:57:05,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6f86920482bb4272a312dd166013771d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6f86920482bb4272a312dd166013771d 2024-11-25T10:57:05,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e386e90f693448c2891f229efa67e9ba to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e386e90f693448c2891f229efa67e9ba 2024-11-25T10:57:05,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/8609d2d46db04c4289f72ee490b4c707 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/8609d2d46db04c4289f72ee490b4c707 2024-11-25T10:57:05,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f3bb396041264294bce3d1c3e886a0b0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f3bb396041264294bce3d1c3e886a0b0 2024-11-25T10:57:05,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a1b50fd7af454013a27a287ea333b61d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a1b50fd7af454013a27a287ea333b61d 2024-11-25T10:57:05,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/add5282f8aec499d80c2b04b2c838217 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/add5282f8aec499d80c2b04b2c838217 2024-11-25T10:57:05,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2dd5b82e77b44e5fb3aabcdfa45d888f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2dd5b82e77b44e5fb3aabcdfa45d888f 2024-11-25T10:57:05,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/70a8d74944074e80a1415a5b112ebcb6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/70a8d74944074e80a1415a5b112ebcb6 2024-11-25T10:57:05,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0b56a269b6834cb3824e733cf2d1ea28 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0b56a269b6834cb3824e733cf2d1ea28 2024-11-25T10:57:05,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2eb8a05ab8604961b922ad45d86872c2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/2eb8a05ab8604961b922ad45d86872c2 2024-11-25T10:57:05,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a17c1e1e5f6a4041bc3abd1f6c365bbe to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/a17c1e1e5f6a4041bc3abd1f6c365bbe 2024-11-25T10:57:05,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/30c5cb08b9c046df9841589978aa7e9f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/30c5cb08b9c046df9841589978aa7e9f 2024-11-25T10:57:05,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee3258b6216a4e4797649366ea777d04 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/ee3258b6216a4e4797649366ea777d04 2024-11-25T10:57:05,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e702d9f89c2c497aa82dabbaf2ac9019 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/e702d9f89c2c497aa82dabbaf2ac9019 2024-11-25T10:57:05,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/be6d135065804d6bac060675b5a39a02 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/be6d135065804d6bac060675b5a39a02 2024-11-25T10:57:05,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/9135cabebabe4dc3b977cbe58474d7dd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/9135cabebabe4dc3b977cbe58474d7dd 2024-11-25T10:57:05,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/0c8bd65c7eaa487a8c4d5b0828ca0b2d 2024-11-25T10:57:05,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/52e388c8ac9e49f29a70ed7fab2ff030, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4c79148a0cc9498aa3e555174f69d375, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ad0cf5b2b9d945d1aa69b4a1d6475202, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ebb421ebd6c9409e8ccf3aa65ed14351, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/c3e5288617ff4039b26dfc54446833b3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/8e2fee8f6e6f455a8aa6077b4828e49e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/af494c60fff04fc383b2e84828d5831d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf960b6fdad5494c8a082efa69421936, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/828cdc9912a04de0ab1a585e6cfdf4f6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/473f6128a5a943349b1d03abc013059d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/12b6658fcf8346529ccd417a6d673ceb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/0ca08961cdf94138adb3d22fce78ad5a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/343d2a3c08da410990482fed24b02677, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7c4a562540e641219a462469b36459d7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bd783befa362408aa98ff3f3f693b15f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f8f3bae05d8e4cbe85d5f6a74a3ee5e5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7b23a44da4a54a5a9fb425f2c49729f3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf18c40235bc4dacbbab1998922d4359, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/108bbcab9bf848f3bc50e6bbc8218ab2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/d73362c55a1848cfadae93aea6732b32, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/2b47aad043c14b46893c5aa2f9cc0934, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4a1a0b58b43b40e1bb2d3512e76b730d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/112af6664cf0478c88c70c0c42bcc6d3] to archive 2024-11-25T10:57:05,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:05,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/52e388c8ac9e49f29a70ed7fab2ff030 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/52e388c8ac9e49f29a70ed7fab2ff030 2024-11-25T10:57:05,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4c79148a0cc9498aa3e555174f69d375 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4c79148a0cc9498aa3e555174f69d375 2024-11-25T10:57:05,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ad0cf5b2b9d945d1aa69b4a1d6475202 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ad0cf5b2b9d945d1aa69b4a1d6475202 2024-11-25T10:57:05,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ebb421ebd6c9409e8ccf3aa65ed14351 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/ebb421ebd6c9409e8ccf3aa65ed14351 2024-11-25T10:57:05,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/c3e5288617ff4039b26dfc54446833b3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/c3e5288617ff4039b26dfc54446833b3 2024-11-25T10:57:05,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/8e2fee8f6e6f455a8aa6077b4828e49e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/8e2fee8f6e6f455a8aa6077b4828e49e 2024-11-25T10:57:05,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/af494c60fff04fc383b2e84828d5831d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/af494c60fff04fc383b2e84828d5831d 2024-11-25T10:57:05,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf960b6fdad5494c8a082efa69421936 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf960b6fdad5494c8a082efa69421936 2024-11-25T10:57:05,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/828cdc9912a04de0ab1a585e6cfdf4f6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/828cdc9912a04de0ab1a585e6cfdf4f6 2024-11-25T10:57:05,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/473f6128a5a943349b1d03abc013059d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/473f6128a5a943349b1d03abc013059d 2024-11-25T10:57:05,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/12b6658fcf8346529ccd417a6d673ceb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/12b6658fcf8346529ccd417a6d673ceb 2024-11-25T10:57:05,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/0ca08961cdf94138adb3d22fce78ad5a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/0ca08961cdf94138adb3d22fce78ad5a 2024-11-25T10:57:05,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/343d2a3c08da410990482fed24b02677 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/343d2a3c08da410990482fed24b02677 2024-11-25T10:57:05,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7c4a562540e641219a462469b36459d7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7c4a562540e641219a462469b36459d7 2024-11-25T10:57:05,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bd783befa362408aa98ff3f3f693b15f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bd783befa362408aa98ff3f3f693b15f 2024-11-25T10:57:05,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f8f3bae05d8e4cbe85d5f6a74a3ee5e5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f8f3bae05d8e4cbe85d5f6a74a3ee5e5 2024-11-25T10:57:05,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7b23a44da4a54a5a9fb425f2c49729f3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/7b23a44da4a54a5a9fb425f2c49729f3 2024-11-25T10:57:05,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf18c40235bc4dacbbab1998922d4359 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/bf18c40235bc4dacbbab1998922d4359 2024-11-25T10:57:05,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/108bbcab9bf848f3bc50e6bbc8218ab2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/108bbcab9bf848f3bc50e6bbc8218ab2 2024-11-25T10:57:05,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/d73362c55a1848cfadae93aea6732b32 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/d73362c55a1848cfadae93aea6732b32 2024-11-25T10:57:05,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/2b47aad043c14b46893c5aa2f9cc0934 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/2b47aad043c14b46893c5aa2f9cc0934 2024-11-25T10:57:05,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4a1a0b58b43b40e1bb2d3512e76b730d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/4a1a0b58b43b40e1bb2d3512e76b730d 2024-11-25T10:57:05,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/112af6664cf0478c88c70c0c42bcc6d3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/112af6664cf0478c88c70c0c42bcc6d3 2024-11-25T10:57:05,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9cccc56e41e44b0aa4f5406afae58017, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/601408bd895a4fba9bdf49517843adaf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c5dc017594854d21ac2fdfe8611dd493, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f0cc9dda35ee45b5a1603baecb4ddb51, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/eba7592dfc144edc8c9518938345d5d2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c7a46e1be3e44baeb0ad95d97a609d0c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/54eb8f85791b430ea69a20439bd6f4d9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9eebefe8a524499fa97a14bfad984ff9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/79d9522623d641ed90594b10dd06a5f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/aa39291411d84814bbe3945103782f67, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/7d03d634d07d4fbc8c928db169bc2274, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f8c95f2df945476f99b11f762fd78062, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c24620cc7b1840e0af8375cd5fa06e34, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2bedcf61e19b4d61ac09aa225cb2ee77, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/125c06bd863a4f3e96a6882394533df3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/bcab3784ac814bd4b8c297ea76afc0cb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/d7b575d757bc474982f5aa24d615c24f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/ea3c685af0b549f4b5c37141a8642f3d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/e82791440abe4e57aa4283cce07645aa, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/93ceaff4be6e49b5bfe274b2fd9c8a2e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/cc182813536e42df87279e48671fca30, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f7b59e4b65d7477a863716e8b35959f1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9774ae40dec647d488ccd6fff53e9107] to archive 2024-11-25T10:57:05,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:05,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9cccc56e41e44b0aa4f5406afae58017 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9cccc56e41e44b0aa4f5406afae58017 2024-11-25T10:57:05,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/601408bd895a4fba9bdf49517843adaf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/601408bd895a4fba9bdf49517843adaf 2024-11-25T10:57:05,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c5dc017594854d21ac2fdfe8611dd493 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c5dc017594854d21ac2fdfe8611dd493 2024-11-25T10:57:05,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f0cc9dda35ee45b5a1603baecb4ddb51 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f0cc9dda35ee45b5a1603baecb4ddb51 2024-11-25T10:57:05,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/eba7592dfc144edc8c9518938345d5d2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/eba7592dfc144edc8c9518938345d5d2 2024-11-25T10:57:05,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c7a46e1be3e44baeb0ad95d97a609d0c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c7a46e1be3e44baeb0ad95d97a609d0c 2024-11-25T10:57:05,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/54eb8f85791b430ea69a20439bd6f4d9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/54eb8f85791b430ea69a20439bd6f4d9 2024-11-25T10:57:05,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9eebefe8a524499fa97a14bfad984ff9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9eebefe8a524499fa97a14bfad984ff9 2024-11-25T10:57:05,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/79d9522623d641ed90594b10dd06a5f7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/79d9522623d641ed90594b10dd06a5f7 2024-11-25T10:57:05,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/aa39291411d84814bbe3945103782f67 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/aa39291411d84814bbe3945103782f67 2024-11-25T10:57:05,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/7d03d634d07d4fbc8c928db169bc2274 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/7d03d634d07d4fbc8c928db169bc2274 2024-11-25T10:57:05,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f8c95f2df945476f99b11f762fd78062 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f8c95f2df945476f99b11f762fd78062 2024-11-25T10:57:05,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c24620cc7b1840e0af8375cd5fa06e34 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/c24620cc7b1840e0af8375cd5fa06e34 2024-11-25T10:57:05,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2bedcf61e19b4d61ac09aa225cb2ee77 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2bedcf61e19b4d61ac09aa225cb2ee77 2024-11-25T10:57:05,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/125c06bd863a4f3e96a6882394533df3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/125c06bd863a4f3e96a6882394533df3 2024-11-25T10:57:05,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/bcab3784ac814bd4b8c297ea76afc0cb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/bcab3784ac814bd4b8c297ea76afc0cb 2024-11-25T10:57:05,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/d7b575d757bc474982f5aa24d615c24f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/d7b575d757bc474982f5aa24d615c24f 2024-11-25T10:57:05,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/ea3c685af0b549f4b5c37141a8642f3d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/ea3c685af0b549f4b5c37141a8642f3d 2024-11-25T10:57:05,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/e82791440abe4e57aa4283cce07645aa to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/e82791440abe4e57aa4283cce07645aa 2024-11-25T10:57:05,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/93ceaff4be6e49b5bfe274b2fd9c8a2e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/93ceaff4be6e49b5bfe274b2fd9c8a2e 2024-11-25T10:57:05,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/cc182813536e42df87279e48671fca30 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/cc182813536e42df87279e48671fca30 2024-11-25T10:57:05,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f7b59e4b65d7477a863716e8b35959f1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/f7b59e4b65d7477a863716e8b35959f1 2024-11-25T10:57:05,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9774ae40dec647d488ccd6fff53e9107 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/9774ae40dec647d488ccd6fff53e9107 2024-11-25T10:57:05,634 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/recovered.edits/443.seqid, newMaxSeqId=443, maxSeqId=4 2024-11-25T10:57:05,635 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. 2024-11-25T10:57:05,635 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for 5dec7eb6196e9e7152a65489b5594c0b: Waiting for close lock at 1732532223699Running coprocessor pre-close hooks at 1732532223699Disabling compacts and flushes for region at 1732532223699Disabling writes for close at 1732532223699Obtaining lock to block concurrent updates at 1732532223699Preparing flush snapshotting stores in 5dec7eb6196e9e7152a65489b5594c0b at 1732532223699Finished memstore snapshotting TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732532223700 (+1 ms)Flushing stores of TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b. at 1732532223700Flushing 5dec7eb6196e9e7152a65489b5594c0b/A: creating writer at 1732532223700Flushing mob file 5dec7eb6196e9e7152a65489b5594c0b/A: appending metadata at 1732532223709 (+9 ms)Flushing mob file 5dec7eb6196e9e7152a65489b5594c0b/A: closing flushed file at 1732532223709Flushing 5dec7eb6196e9e7152a65489b5594c0b/A: appending metadata at 1732532224185 (+476 ms)Flushing 5dec7eb6196e9e7152a65489b5594c0b/A: closing flushed file at 1732532224185Flushing 5dec7eb6196e9e7152a65489b5594c0b/B: creating writer at 1732532224637 (+452 ms)Flushing 5dec7eb6196e9e7152a65489b5594c0b/B: appending metadata at 1732532224642 (+5 ms)Flushing 5dec7eb6196e9e7152a65489b5594c0b/B: closing flushed file at 1732532224642Flushing 5dec7eb6196e9e7152a65489b5594c0b/C: creating writer at 1732532225086 (+444 ms)Flushing 5dec7eb6196e9e7152a65489b5594c0b/C: appending metadata at 1732532225090 (+4 ms)Flushing 5dec7eb6196e9e7152a65489b5594c0b/C: closing flushed file at 1732532225090Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e5280eb: reopening flushed file at 1732532225510 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cc19a01: reopening flushed file at 1732532225516 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46edd86d: reopening flushed file at 1732532225522 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5dec7eb6196e9e7152a65489b5594c0b in 1830ms, sequenceid=440, compaction requested=true at 1732532225529 (+7 ms)Writing region close event to WAL at 1732532225631 (+102 ms)Running coprocessor post-close hooks at 1732532225635 (+4 ms)Closed at 1732532225635 2024-11-25T10:57:05,637 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed 5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:05,638 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=5dec7eb6196e9e7152a65489b5594c0b, regionState=CLOSED 2024-11-25T10:57:05,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:57:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T10:57:05,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-25T10:57:05,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure 5dec7eb6196e9e7152a65489b5594c0b, server=6bfc37bad115,44671,1732532110409 in 2.1040 sec 2024-11-25T10:57:05,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-25T10:57:05,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5dec7eb6196e9e7152a65489b5594c0b, UNASSIGN in 2.1150 sec 2024-11-25T10:57:05,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-25T10:57:05,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1230 sec 2024-11-25T10:57:05,654 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532225653"}]},"ts":"1732532225653"} 2024-11-25T10:57:05,655 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T10:57:05,655 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T10:57:05,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1770 sec 2024-11-25T10:57:07,466 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T10:57:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T10:57:07,664 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:07,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T10:57:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:07,667 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T10:57:07,668 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:07,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,673 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/recovered.edits] 2024-11-25T10:57:07,676 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6d91d93f1a9149adb8cd4f5df31c11c6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/6d91d93f1a9149adb8cd4f5df31c11c6 2024-11-25T10:57:07,677 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/c6277cec16484dbcbcd4513e91e08623 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/c6277cec16484dbcbcd4513e91e08623 2024-11-25T10:57:07,678 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f48d8541a4e8437b8d447bb4557ff984 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/A/f48d8541a4e8437b8d447bb4557ff984 2024-11-25T10:57:07,681 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/11677151f1c445d7933bed2304e7ac67 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/11677151f1c445d7933bed2304e7ac67 2024-11-25T10:57:07,682 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/19242dfb183d439f9c8e4abb02398f15 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/19242dfb183d439f9c8e4abb02398f15 2024-11-25T10:57:07,684 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f356b048f4cd483ab21204ee807fef57 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/B/f356b048f4cd483ab21204ee807fef57 2024-11-25T10:57:07,686 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2e2922efa96f49f298ba6aadc8ae5775 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/2e2922efa96f49f298ba6aadc8ae5775 2024-11-25T10:57:07,687 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/31004cd2846a42c3a408c0f47c3c298f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/31004cd2846a42c3a408c0f47c3c298f 2024-11-25T10:57:07,688 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/b825d20bf9ea46289f737547ac2d5e05 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/C/b825d20bf9ea46289f737547ac2d5e05 2024-11-25T10:57:07,690 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/recovered.edits/443.seqid to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b/recovered.edits/443.seqid 2024-11-25T10:57:07,691 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,691 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T10:57:07,692 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T10:57:07,692 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T10:57:07,696 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112530116cec2dca40cfb130d7925c3c1447_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112530116cec2dca40cfb130d7925c3c1447_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,697 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253501dd6ef6484909872ec376b3d60459_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253501dd6ef6484909872ec376b3d60459_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,699 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112540287d5b1c7c4f2691731d09ae18f689_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112540287d5b1c7c4f2691731d09ae18f689_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,700 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125441ef0cae5ae46029cfd3bafd1e03414_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125441ef0cae5ae46029cfd3bafd1e03414_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,701 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112552e3ca683dbd43ce84e6557177b0ec5a_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112552e3ca683dbd43ce84e6557177b0ec5a_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,702 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112567da03a2703a4e338e8b1abb70edaec2_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112567da03a2703a4e338e8b1abb70edaec2_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,703 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257278891fee0e4b1386948ac9df2ffa84_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257278891fee0e4b1386948ac9df2ffa84_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,704 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112576b759d4f54741df9a01310f466cc82e_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112576b759d4f54741df9a01310f466cc82e_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,705 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258fd3eb0704c1454aaf3fc482246a0290_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258fd3eb0704c1454aaf3fc482246a0290_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,706 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259ae2748e3c1240f792013417b41e879b_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259ae2748e3c1240f792013417b41e879b_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,707 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a67bab0d1b0d4deb9c0072aa97923041_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a67bab0d1b0d4deb9c0072aa97923041_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,709 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aa0ce553028b4873a186e5c348c0d067_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aa0ce553028b4873a186e5c348c0d067_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,710 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aaa6991b5dcc4c5f9fa5e5e455dc7f0d_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125aaa6991b5dcc4c5f9fa5e5e455dc7f0d_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,711 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae6ad5f52297494fbf6eeb34eb5366ca_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae6ad5f52297494fbf6eeb34eb5366ca_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,712 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bab3fb6506e94baebadd558dcb1c2b3b_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bab3fb6506e94baebadd558dcb1c2b3b_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,713 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bce3b9086a994b6294111eb3efee2bf9_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bce3b9086a994b6294111eb3efee2bf9_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,714 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d874993a045f49e4bd51483b1e16b707_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d874993a045f49e4bd51483b1e16b707_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,715 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e3b2a40f76ec4cb18e59bf3667fcf038_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e3b2a40f76ec4cb18e59bf3667fcf038_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,716 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125fc8a059911654a689682655209a0bee5_5dec7eb6196e9e7152a65489b5594c0b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125fc8a059911654a689682655209a0bee5_5dec7eb6196e9e7152a65489b5594c0b 2024-11-25T10:57:07,717 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T10:57:07,719 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:07,722 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T10:57:07,724 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T10:57:07,727 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:07,727 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T10:57:07,728 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732532227728"}]},"ts":"9223372036854775807"} 2024-11-25T10:57:07,730 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T10:57:07,730 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5dec7eb6196e9e7152a65489b5594c0b, NAME => 'TestAcidGuarantees,,1732532198910.5dec7eb6196e9e7152a65489b5594c0b.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T10:57:07,730 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T10:57:07,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732532227730"}]},"ts":"9223372036854775807"} 2024-11-25T10:57:07,732 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T10:57:07,733 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:07,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-25T10:57:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T10:57:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T10:57:07,775 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:07,788 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 251), OpenFileDescriptor=459 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1401 (was 1539), ProcessCount=11 (was 11), AvailableMemoryMB=689 (was 1215) 2024-11-25T10:57:07,801 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1401, ProcessCount=11, AvailableMemoryMB=688 2024-11-25T10:57:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:57:07,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:57:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:07,805 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T10:57:07,805 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:07,805 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-25T10:57:07,806 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T10:57:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T10:57:07,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742230_1406 (size=1061) 2024-11-25T10:57:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T10:57:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T10:57:08,215 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:57:08,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742231_1407 (size=53) 2024-11-25T10:57:08,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:57:08,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e529e4988658b515c2e6b809e2e4ac97, disabling compactions & flushes 2024-11-25T10:57:08,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. after waiting 0 ms 2024-11-25T10:57:08,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e529e4988658b515c2e6b809e2e4ac97: Waiting for close lock at 1732532228221Disabling compacts and flushes for region at 1732532228221Disabling writes for close at 1732532228221Writing region close event to WAL at 1732532228221Closed at 1732532228221 2024-11-25T10:57:08,222 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T10:57:08,223 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732532228223"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532228223"}]},"ts":"1732532228223"} 2024-11-25T10:57:08,224 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T10:57:08,225 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T10:57:08,225 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532228225"}]},"ts":"1732532228225"} 2024-11-25T10:57:08,227 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T10:57:08,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, ASSIGN}] 2024-11-25T10:57:08,228 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, ASSIGN 2024-11-25T10:57:08,229 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, ASSIGN; state=OFFLINE, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=false 2024-11-25T10:57:08,380 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=e529e4988658b515c2e6b809e2e4ac97, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:08,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, ASSIGN because future has completed 2024-11-25T10:57:08,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:57:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T10:57:08,580 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,581 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:57:08,582 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,582 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:57:08,582 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,582 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,597 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,609 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:08,609 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e529e4988658b515c2e6b809e2e4ac97 columnFamilyName A 2024-11-25T10:57:08,609 DEBUG [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:08,610 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.HStore(327): Store=e529e4988658b515c2e6b809e2e4ac97/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:08,610 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,611 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:08,611 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e529e4988658b515c2e6b809e2e4ac97 columnFamilyName B 2024-11-25T10:57:08,611 DEBUG [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:08,612 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.HStore(327): Store=e529e4988658b515c2e6b809e2e4ac97/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:08,612 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,613 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:08,613 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e529e4988658b515c2e6b809e2e4ac97 columnFamilyName C 2024-11-25T10:57:08,613 DEBUG [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:08,614 INFO [StoreOpener-e529e4988658b515c2e6b809e2e4ac97-1 {}] regionserver.HStore(327): Store=e529e4988658b515c2e6b809e2e4ac97/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:08,614 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,614 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,615 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,615 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,616 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,616 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,616 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:57:08,621 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,628 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:57:08,629 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened e529e4988658b515c2e6b809e2e4ac97; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68649697, jitterRate=0.022960200905799866}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:57:08,629 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:08,630 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for e529e4988658b515c2e6b809e2e4ac97: Running coprocessor pre-open hook at 1732532228582Writing region info on filesystem at 1732532228582Initializing all the Stores at 1732532228595 (+13 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532228595Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532228596 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532228596Cleaning up temporary data from old regions at 1732532228616 (+20 ms)Running coprocessor post-open hooks at 1732532228629 (+13 ms)Region opened successfully at 1732532228630 (+1 ms) 2024-11-25T10:57:08,631 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., pid=110, masterSystemTime=1732532228536 2024-11-25T10:57:08,633 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,634 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:08,634 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=e529e4988658b515c2e6b809e2e4ac97, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:08,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:57:08,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-25T10:57:08,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 in 256 msec 2024-11-25T10:57:08,644 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T10:57:08,644 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532228644"}]},"ts":"1732532228644"} 2024-11-25T10:57:08,646 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T10:57:08,647 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T10:57:08,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 844 msec 2024-11-25T10:57:08,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-25T10:57:08,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, ASSIGN in 413 msec 2024-11-25T10:57:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-25T10:57:08,944 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:08,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:08,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:08,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3568cff0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:08,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:08,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d13cc6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,950 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:08,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:08,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T10:57:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39013,1732532109078 2024-11-25T10:57:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@207ec030 2024-11-25T10:57:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T10:57:08,957 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T10:57:08,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:08,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:08,960 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:08,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:08,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,961 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:08,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:08,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f2dc990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:08,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:08,977 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60c2efde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:08,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216ddebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:08,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:08,994 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3311ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:08,996 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55eae1f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:08,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:08,999 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4469d8e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:09,000 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e2da0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:09,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:09,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9ca61f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:09,008 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bd2a88e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:09,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:09,012 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c94c0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:09,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:09,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:09,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7e37bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:09,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:09,029 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b54d999, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:09,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:09,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b944bca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:09,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:09,049 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa28187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:09,056 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74de0d0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:09,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:09,064 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@786eb64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:09,070 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3f4089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:09,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:09,081 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:09,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:09,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:09,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,082 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:09,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:09,083 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:09,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c1b90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:09,090 DEBUG [Thread-1855 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,093 DEBUG [Thread-1846 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,094 DEBUG [Thread-1846 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,094 DEBUG [Thread-1844 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,094 DEBUG [Thread-1846 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,095 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,096 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,101 DEBUG [Thread-1850 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,102 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-25T10:57:09,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,103 DEBUG [Thread-1852 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T10:57:09,107 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:09,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:09,108 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,110 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,112 DEBUG [Thread-1848 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,114 DEBUG [Thread-1863 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,114 DEBUG [Thread-1863 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T10:57:09,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,122 DEBUG [Thread-1857 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:09,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:09,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:09,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:09,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:09,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:09,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:09,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2] 2024-11-25T10:57:09,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/bbcf6c3c4c5045d4b418ce536a17fe3c is 50, key is test_row_0/A:col10/1732532229113/Put/seqid=0 2024-11-25T10:57:09,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742232_1408 (size=12001) 2024-11-25T10:57:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T10:57:09,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/bbcf6c3c4c5045d4b418ce536a17fe3c 2024-11-25T10:57:09,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532229170, max timestamp=1732532229236Num uniques 859; 2024-11-25T10:57:09,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:09,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532229170, max timestamp=1732532229236Num uniques 859; 2024-11-25T10:57:09,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:09,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532229170, max timestamp=1732532229236Num uniques 859; 2024-11-25T10:57:09,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:09,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T10:57:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/80aa54508fb943fdb9db9a982aba14f8 is 50, key is test_row_0/B:col10/1732532229113/Put/seqid=0 2024-11-25T10:57:09,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742233_1409 (size=12001) 2024-11-25T10:57:09,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/80aa54508fb943fdb9db9a982aba14f8 2024-11-25T10:57:09,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532289321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532289321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532289323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532289327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532289329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/4524bce31e484d5db669e4558c511d28 is 50, key is test_row_0/C:col10/1732532229113/Put/seqid=0 2024-11-25T10:57:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T10:57:09,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T10:57:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742234_1410 (size=12001) 2024-11-25T10:57:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532289454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532289454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532289455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532289455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532289455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T10:57:09,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:09,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532289673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532289675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532289675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532289675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532289677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:09,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T10:57:09,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T10:57:09,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:09,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:09,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/4524bce31e484d5db669e4558c511d28 2024-11-25T10:57:09,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/bbcf6c3c4c5045d4b418ce536a17fe3c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/bbcf6c3c4c5045d4b418ce536a17fe3c 2024-11-25T10:57:09,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/bbcf6c3c4c5045d4b418ce536a17fe3c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T10:57:09,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/80aa54508fb943fdb9db9a982aba14f8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/80aa54508fb943fdb9db9a982aba14f8 2024-11-25T10:57:09,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/80aa54508fb943fdb9db9a982aba14f8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T10:57:09,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/4524bce31e484d5db669e4558c511d28 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4524bce31e484d5db669e4558c511d28 2024-11-25T10:57:09,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4524bce31e484d5db669e4558c511d28, entries=150, sequenceid=13, filesize=11.7 K 2024-11-25T10:57:09,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for e529e4988658b515c2e6b809e2e4ac97 in 772ms, sequenceid=13, compaction requested=false 2024-11-25T10:57:09,890 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T10:57:09,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:09,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T10:57:09,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:09,901 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-25T10:57:09,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/5437975c5e294ed080aed487b5d86873 is 50, key is test_row_0/A:col10/1732532229327/Put/seqid=0 2024-11-25T10:57:09,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742235_1411 (size=12001) 2024-11-25T10:57:09,934 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/5437975c5e294ed080aed487b5d86873 2024-11-25T10:57:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/6eec3c0cd757403ba34df34d68e29b90 is 50, key is test_row_0/B:col10/1732532229327/Put/seqid=0 2024-11-25T10:57:09,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742236_1412 (size=12001) 2024-11-25T10:57:09,987 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/6eec3c0cd757403ba34df34d68e29b90 2024-11-25T10:57:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/8c2c4a38cf3849758f97d97e247a53a0 is 50, key is test_row_0/C:col10/1732532229327/Put/seqid=0 2024-11-25T10:57:10,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:10,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742237_1413 (size=12001) 2024-11-25T10:57:10,014 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/8c2c4a38cf3849758f97d97e247a53a0 2024-11-25T10:57:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/5437975c5e294ed080aed487b5d86873 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/5437975c5e294ed080aed487b5d86873 2024-11-25T10:57:10,029 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/5437975c5e294ed080aed487b5d86873, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T10:57:10,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/6eec3c0cd757403ba34df34d68e29b90 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6eec3c0cd757403ba34df34d68e29b90 2024-11-25T10:57:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532290021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532290022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532290023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532290023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532290024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,054 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6eec3c0cd757403ba34df34d68e29b90, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T10:57:10,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/8c2c4a38cf3849758f97d97e247a53a0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/8c2c4a38cf3849758f97d97e247a53a0 2024-11-25T10:57:10,072 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/8c2c4a38cf3849758f97d97e247a53a0, entries=150, sequenceid=47, filesize=11.7 K 2024-11-25T10:57:10,074 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for e529e4988658b515c2e6b809e2e4ac97 in 173ms, sequenceid=47, compaction requested=false 2024-11-25T10:57:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-25T10:57:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-25T10:57:10,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-25T10:57:10,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 977 msec 2024-11-25T10:57:10,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 986 msec 2024-11-25T10:57:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:10,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T10:57:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:10,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cd6efc596c774854a7a23ceac89c170e is 50, key is test_row_0/A:col10/1732532230149/Put/seqid=0 2024-11-25T10:57:10,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742238_1414 (size=12001) 2024-11-25T10:57:10,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cd6efc596c774854a7a23ceac89c170e 2024-11-25T10:57:10,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/e66e00f8b57b4afb80746b78fdb1460e is 50, key is test_row_0/B:col10/1732532230149/Put/seqid=0 2024-11-25T10:57:10,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742239_1415 (size=12001) 2024-11-25T10:57:10,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/e66e00f8b57b4afb80746b78fdb1460e 2024-11-25T10:57:10,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/fee2eb731ca142a09785a7377c5a6357 is 50, key is test_row_0/C:col10/1732532230149/Put/seqid=0 2024-11-25T10:57:10,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742240_1416 (size=12001) 2024-11-25T10:57:10,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/fee2eb731ca142a09785a7377c5a6357 2024-11-25T10:57:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T10:57:10,244 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:10,244 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-25T10:57:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T10:57:10,247 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:10,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:10,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:10,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cd6efc596c774854a7a23ceac89c170e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd6efc596c774854a7a23ceac89c170e 2024-11-25T10:57:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532290221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd6efc596c774854a7a23ceac89c170e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T10:57:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532290221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/e66e00f8b57b4afb80746b78fdb1460e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e66e00f8b57b4afb80746b78fdb1460e 2024-11-25T10:57:10,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532290224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532290225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532290225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e66e00f8b57b4afb80746b78fdb1460e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T10:57:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/fee2eb731ca142a09785a7377c5a6357 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fee2eb731ca142a09785a7377c5a6357 2024-11-25T10:57:10,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fee2eb731ca142a09785a7377c5a6357, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T10:57:10,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for e529e4988658b515c2e6b809e2e4ac97 in 126ms, sequenceid=61, compaction requested=true 2024-11-25T10:57:10,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:10,279 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:10,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:10,281 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:10,281 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:10,281 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:10,281 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,281 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/bbcf6c3c4c5045d4b418ce536a17fe3c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/5437975c5e294ed080aed487b5d86873, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd6efc596c774854a7a23ceac89c170e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=35.2 K 2024-11-25T10:57:10,282 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbcf6c3c4c5045d4b418ce536a17fe3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732532229113 2024-11-25T10:57:10,282 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5437975c5e294ed080aed487b5d86873, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532229249 2024-11-25T10:57:10,283 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:10,283 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:10,283 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,283 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/80aa54508fb943fdb9db9a982aba14f8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6eec3c0cd757403ba34df34d68e29b90, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e66e00f8b57b4afb80746b78fdb1460e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=35.2 K 2024-11-25T10:57:10,283 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6efc596c774854a7a23ceac89c170e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532230018 2024-11-25T10:57:10,297 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 80aa54508fb943fdb9db9a982aba14f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732532229113 2024-11-25T10:57:10,297 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eec3c0cd757403ba34df34d68e29b90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532229249 2024-11-25T10:57:10,301 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e66e00f8b57b4afb80746b78fdb1460e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532230018 2024-11-25T10:57:10,343 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:10,344 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/8d5d8725d0444a969ed4778a594e0ba2 is 50, key is test_row_0/A:col10/1732532230149/Put/seqid=0 2024-11-25T10:57:10,349 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:10,349 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/145cfc7641d04636aa407f4b35b1a2ac is 50, key is test_row_0/B:col10/1732532230149/Put/seqid=0 2024-11-25T10:57:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T10:57:10,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532230217, max timestamp=1732532230377Num uniques 859; 2024-11-25T10:57:10,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532230217, max timestamp=1732532230377Num uniques 859; 2024-11-25T10:57:10,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:10,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:10,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-25T10:57:10,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:10,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:10,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:10,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:10,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:10,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:10,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742241_1417 (size=12104) 2024-11-25T10:57:10,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/40e0372647934e16b4e7df15c463bbc1 is 50, key is test_row_0/A:col10/1732532230383/Put/seqid=0 2024-11-25T10:57:10,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:10,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:10,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742242_1418 (size=12104) 2024-11-25T10:57:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742243_1419 (size=16681) 2024-11-25T10:57:10,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/40e0372647934e16b4e7df15c463bbc1 2024-11-25T10:57:10,427 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/8d5d8725d0444a969ed4778a594e0ba2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/8d5d8725d0444a969ed4778a594e0ba2 2024-11-25T10:57:10,428 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/145cfc7641d04636aa407f4b35b1a2ac as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/145cfc7641d04636aa407f4b35b1a2ac 2024-11-25T10:57:10,437 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into 8d5d8725d0444a969ed4778a594e0ba2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:10,437 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:10,437 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=13, startTime=1732532230279; duration=0sec 2024-11-25T10:57:10,437 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:10,437 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:10,437 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:10,438 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:10,438 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:10,438 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,438 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4524bce31e484d5db669e4558c511d28, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/8c2c4a38cf3849758f97d97e247a53a0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fee2eb731ca142a09785a7377c5a6357] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=35.2 K 2024-11-25T10:57:10,439 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4524bce31e484d5db669e4558c511d28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732532229113 2024-11-25T10:57:10,439 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c2c4a38cf3849758f97d97e247a53a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732532229249 2024-11-25T10:57:10,439 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fee2eb731ca142a09785a7377c5a6357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532230018 2024-11-25T10:57:10,444 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into 145cfc7641d04636aa407f4b35b1a2ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:10,444 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:10,444 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=13, startTime=1732532230281; duration=0sec 2024-11-25T10:57:10,444 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:10,444 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:10,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5ecee5b4d9754c85a3d25f03ab49f8de is 50, key is test_row_0/B:col10/1732532230383/Put/seqid=0 2024-11-25T10:57:10,450 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:10,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,451 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/ed83e57ec5604cf6b84957a3f73b78aa is 50, key is test_row_0/C:col10/1732532230149/Put/seqid=0 2024-11-25T10:57:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532290432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532290441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532290443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532290446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532290451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742244_1420 (size=12001) 2024-11-25T10:57:10,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742245_1421 (size=12104) 2024-11-25T10:57:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T10:57:10,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532290564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532290564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532290574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532290574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532290575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532290784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532290784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532290784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532290784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532290784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T10:57:10,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:10,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:10,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5ecee5b4d9754c85a3d25f03ab49f8de 2024-11-25T10:57:10,916 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/ed83e57ec5604cf6b84957a3f73b78aa as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ed83e57ec5604cf6b84957a3f73b78aa 2024-11-25T10:57:10,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/5a9c06c1d7954766964d18e31b44b2f9 is 50, key is test_row_0/C:col10/1732532230383/Put/seqid=0 2024-11-25T10:57:10,928 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into ed83e57ec5604cf6b84957a3f73b78aa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:10,928 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:10,928 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=13, startTime=1732532230281; duration=0sec 2024-11-25T10:57:10,928 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:10,928 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:10,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742246_1422 (size=12001) 2024-11-25T10:57:11,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:11,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532291104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532291104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532291104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532291105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532291105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,176 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T10:57:11,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:11,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/5a9c06c1d7954766964d18e31b44b2f9 2024-11-25T10:57:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/40e0372647934e16b4e7df15c463bbc1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/40e0372647934e16b4e7df15c463bbc1 2024-11-25T10:57:11,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:11,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:11,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:11,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:11,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T10:57:11,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/40e0372647934e16b4e7df15c463bbc1, entries=250, sequenceid=83, filesize=16.3 K 2024-11-25T10:57:11,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5ecee5b4d9754c85a3d25f03ab49f8de as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ecee5b4d9754c85a3d25f03ab49f8de 2024-11-25T10:57:11,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ecee5b4d9754c85a3d25f03ab49f8de, entries=150, sequenceid=83, filesize=11.7 K 2024-11-25T10:57:11,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/5a9c06c1d7954766964d18e31b44b2f9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/5a9c06c1d7954766964d18e31b44b2f9 2024-11-25T10:57:11,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/5a9c06c1d7954766964d18e31b44b2f9, entries=150, sequenceid=83, filesize=11.7 K 2024-11-25T10:57:11,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for e529e4988658b515c2e6b809e2e4ac97 in 1038ms, sequenceid=83, compaction requested=false 2024-11-25T10:57:11,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:11,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T10:57:11,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:11,528 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T10:57:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:11,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/ca6eddbc813844feb351372e34868d2f is 50, key is test_row_0/A:col10/1732532230428/Put/seqid=0 2024-11-25T10:57:11,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742247_1423 (size=12001) 2024-11-25T10:57:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:11,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532291639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532291640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532291650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532291651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532291652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532291764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532291764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532291765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532291772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532291780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,961 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/ca6eddbc813844feb351372e34868d2f 2024-11-25T10:57:11,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532291984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/1d801460a439467aafbe31feb7776be2 is 50, key is test_row_0/B:col10/1732532230428/Put/seqid=0 2024-11-25T10:57:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532291984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532291984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532292004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532292004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742248_1424 (size=12001) 2024-11-25T10:57:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532292299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532292305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532292305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532292323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532292323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T10:57:12,433 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/1d801460a439467aafbe31feb7776be2 2024-11-25T10:57:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/f88857dd61cc495da27127dc62b07a20 is 50, key is test_row_0/C:col10/1732532230428/Put/seqid=0 2024-11-25T10:57:12,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742249_1425 (size=12001) 2024-11-25T10:57:12,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532292813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532292824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532292824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532292843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532292843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:12,934 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/f88857dd61cc495da27127dc62b07a20 2024-11-25T10:57:12,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/ca6eddbc813844feb351372e34868d2f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/ca6eddbc813844feb351372e34868d2f 2024-11-25T10:57:12,947 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/ca6eddbc813844feb351372e34868d2f, entries=150, sequenceid=106, filesize=11.7 K 2024-11-25T10:57:12,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/1d801460a439467aafbe31feb7776be2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d801460a439467aafbe31feb7776be2 2024-11-25T10:57:12,962 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d801460a439467aafbe31feb7776be2, entries=150, sequenceid=106, filesize=11.7 K 2024-11-25T10:57:12,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/f88857dd61cc495da27127dc62b07a20 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f88857dd61cc495da27127dc62b07a20 2024-11-25T10:57:12,967 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f88857dd61cc495da27127dc62b07a20, entries=150, sequenceid=106, filesize=11.7 K 2024-11-25T10:57:12,968 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e529e4988658b515c2e6b809e2e4ac97 in 1440ms, sequenceid=106, compaction requested=true 2024-11-25T10:57:12,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:12,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:12,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-25T10:57:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-25T10:57:12,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-25T10:57:12,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7220 sec 2024-11-25T10:57:12,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.7280 sec 2024-11-25T10:57:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:13,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:57:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:13,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/001efef2d2fe4dd7a7a23b50da5d4fa7 is 50, key is test_row_0/A:col10/1732532231649/Put/seqid=0 2024-11-25T10:57:13,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742250_1426 (size=14341) 2024-11-25T10:57:13,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/001efef2d2fe4dd7a7a23b50da5d4fa7 2024-11-25T10:57:13,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532293917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532293935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532293944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:13,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532293951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:13,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532293956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:13,997 WARN [regionserver/6bfc37bad115:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-25T10:57:14,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/bbcf6c3c4c5045d4b418ce536a17fe3c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/5437975c5e294ed080aed487b5d86873, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd6efc596c774854a7a23ceac89c170e] to archive 2024-11-25T10:57:14,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5ba11db194714b2b85ef753b1b0e0b4b is 50, key is test_row_0/B:col10/1732532231649/Put/seqid=0 2024-11-25T10:57:14,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:14,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/bbcf6c3c4c5045d4b418ce536a17fe3c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/bbcf6c3c4c5045d4b418ce536a17fe3c 2024-11-25T10:57:14,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/5437975c5e294ed080aed487b5d86873 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/5437975c5e294ed080aed487b5d86873 2024-11-25T10:57:14,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd6efc596c774854a7a23ceac89c170e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd6efc596c774854a7a23ceac89c170e 2024-11-25T10:57:14,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742251_1427 (size=12001) 2024-11-25T10:57:14,041 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/80aa54508fb943fdb9db9a982aba14f8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6eec3c0cd757403ba34df34d68e29b90, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e66e00f8b57b4afb80746b78fdb1460e] to archive 2024-11-25T10:57:14,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5ba11db194714b2b85ef753b1b0e0b4b 2024-11-25T10:57:14,046 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532294057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,069 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/80aa54508fb943fdb9db9a982aba14f8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/80aa54508fb943fdb9db9a982aba14f8 2024-11-25T10:57:14,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6eec3c0cd757403ba34df34d68e29b90 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6eec3c0cd757403ba34df34d68e29b90 2024-11-25T10:57:14,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e66e00f8b57b4afb80746b78fdb1460e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e66e00f8b57b4afb80746b78fdb1460e 2024-11-25T10:57:14,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4524bce31e484d5db669e4558c511d28, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/8c2c4a38cf3849758f97d97e247a53a0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fee2eb731ca142a09785a7377c5a6357] to archive 2024-11-25T10:57:14,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532294085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532294085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532294085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532294085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4524bce31e484d5db669e4558c511d28 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4524bce31e484d5db669e4558c511d28 2024-11-25T10:57:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/9b6b00ae0bdf4e93b707f8eb23a9b466 is 50, key is test_row_0/C:col10/1732532231649/Put/seqid=0 2024-11-25T10:57:14,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/8c2c4a38cf3849758f97d97e247a53a0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/8c2c4a38cf3849758f97d97e247a53a0 2024-11-25T10:57:14,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fee2eb731ca142a09785a7377c5a6357 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fee2eb731ca142a09785a7377c5a6357 2024-11-25T10:57:14,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742252_1428 (size=12001) 2024-11-25T10:57:14,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/9b6b00ae0bdf4e93b707f8eb23a9b466 2024-11-25T10:57:14,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/001efef2d2fe4dd7a7a23b50da5d4fa7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/001efef2d2fe4dd7a7a23b50da5d4fa7 2024-11-25T10:57:14,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/001efef2d2fe4dd7a7a23b50da5d4fa7, entries=200, sequenceid=120, filesize=14.0 K 2024-11-25T10:57:14,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5ba11db194714b2b85ef753b1b0e0b4b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ba11db194714b2b85ef753b1b0e0b4b 2024-11-25T10:57:14,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532294279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ba11db194714b2b85ef753b1b0e0b4b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-25T10:57:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/9b6b00ae0bdf4e93b707f8eb23a9b466 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9b6b00ae0bdf4e93b707f8eb23a9b466 2024-11-25T10:57:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532294306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532294306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532294306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532294307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9b6b00ae0bdf4e93b707f8eb23a9b466, entries=150, sequenceid=120, filesize=11.7 K 2024-11-25T10:57:14,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e529e4988658b515c2e6b809e2e4ac97 in 499ms, sequenceid=120, compaction requested=true 2024-11-25T10:57:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:57:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:57:14,357 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:14,357 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:14,376 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:14,377 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:14,377 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:14,377 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ed83e57ec5604cf6b84957a3f73b78aa, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/5a9c06c1d7954766964d18e31b44b2f9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f88857dd61cc495da27127dc62b07a20, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9b6b00ae0bdf4e93b707f8eb23a9b466] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=47.0 K 2024-11-25T10:57:14,377 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:14,377 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:14,377 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:14,377 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/8d5d8725d0444a969ed4778a594e0ba2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/40e0372647934e16b4e7df15c463bbc1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/ca6eddbc813844feb351372e34868d2f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/001efef2d2fe4dd7a7a23b50da5d4fa7] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=53.8 K 2024-11-25T10:57:14,378 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ed83e57ec5604cf6b84957a3f73b78aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532230018 2024-11-25T10:57:14,383 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d5d8725d0444a969ed4778a594e0ba2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532230018 2024-11-25T10:57:14,384 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9c06c1d7954766964d18e31b44b2f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732532230217 2024-11-25T10:57:14,384 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e0372647934e16b4e7df15c463bbc1, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732532230217 2024-11-25T10:57:14,384 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f88857dd61cc495da27127dc62b07a20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732532230428 2024-11-25T10:57:14,388 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6eddbc813844feb351372e34868d2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732532230428 2024-11-25T10:57:14,392 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001efef2d2fe4dd7a7a23b50da5d4fa7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732532231638 2024-11-25T10:57:14,393 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b6b00ae0bdf4e93b707f8eb23a9b466, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732532231649 2024-11-25T10:57:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T10:57:14,405 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:14,406 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-25T10:57:14,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T10:57:14,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:14,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:14,423 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:14,424 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/f47e004ceff940189b0244da6ac6fff7 is 50, key is test_row_0/C:col10/1732532231649/Put/seqid=0 2024-11-25T10:57:14,439 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:14,440 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/43e07169859e49d794e0eae16f8f0e1c is 50, key is test_row_0/A:col10/1732532231649/Put/seqid=0 2024-11-25T10:57:14,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742253_1429 (size=12139) 2024-11-25T10:57:14,495 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/f47e004ceff940189b0244da6ac6fff7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f47e004ceff940189b0244da6ac6fff7 2024-11-25T10:57:14,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742254_1430 (size=12139) 2024-11-25T10:57:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T10:57:14,533 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into f47e004ceff940189b0244da6ac6fff7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:14,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:14,533 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=12, startTime=1732532234357; duration=0sec 2024-11-25T10:57:14,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:14,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:14,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:14,576 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:14,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T10:57:14,577 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:14,577 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:14,577 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/145cfc7641d04636aa407f4b35b1a2ac, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ecee5b4d9754c85a3d25f03ab49f8de, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d801460a439467aafbe31feb7776be2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ba11db194714b2b85ef753b1b0e0b4b] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=47.0 K 2024-11-25T10:57:14,577 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:57:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:14,583 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 145cfc7641d04636aa407f4b35b1a2ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732532230018 2024-11-25T10:57:14,588 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ecee5b4d9754c85a3d25f03ab49f8de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732532230217 2024-11-25T10:57:14,592 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d801460a439467aafbe31feb7776be2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732532230428 2024-11-25T10:57:14,595 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/43e07169859e49d794e0eae16f8f0e1c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/43e07169859e49d794e0eae16f8f0e1c 2024-11-25T10:57:14,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/d6bb9e57caff403381243372a4b360a7 is 50, key is test_row_0/A:col10/1732532233942/Put/seqid=0 2024-11-25T10:57:14,603 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba11db194714b2b85ef753b1b0e0b4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732532231649 2024-11-25T10:57:14,625 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into 43e07169859e49d794e0eae16f8f0e1c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:14,625 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:14,625 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=12, startTime=1732532234357; duration=0sec 2024-11-25T10:57:14,625 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:14,625 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:14,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:14,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742255_1431 (size=12151) 2024-11-25T10:57:14,652 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/d6bb9e57caff403381243372a4b360a7 2024-11-25T10:57:14,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/2bd83a77e16d43abbb867e23dd1c01c5 is 50, key is test_row_0/B:col10/1732532233942/Put/seqid=0 2024-11-25T10:57:14,681 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:14,681 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5e15828073af4976ae1117d87d02e993 is 50, key is test_row_0/B:col10/1732532231649/Put/seqid=0 2024-11-25T10:57:14,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742256_1432 (size=12151) 2024-11-25T10:57:14,689 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/2bd83a77e16d43abbb867e23dd1c01c5 2024-11-25T10:57:14,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742257_1433 (size=12139) 2024-11-25T10:57:14,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/bf4fc60eae214a0bb36f370b22385ad6 is 50, key is test_row_0/C:col10/1732532233942/Put/seqid=0 2024-11-25T10:57:14,709 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/5e15828073af4976ae1117d87d02e993 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5e15828073af4976ae1117d87d02e993 2024-11-25T10:57:14,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742258_1434 (size=12151) 2024-11-25T10:57:14,724 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into 5e15828073af4976ae1117d87d02e993(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:14,724 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:14,725 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=12, startTime=1732532234357; duration=0sec 2024-11-25T10:57:14,725 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:14,725 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T10:57:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532294733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532294748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532294750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532294754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532294757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532294865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532294866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532294866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532294881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532294885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:14,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T10:57:15,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532295093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532295097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532295097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532295105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,125 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/bf4fc60eae214a0bb36f370b22385ad6 2024-11-25T10:57:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532295120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/d6bb9e57caff403381243372a4b360a7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d6bb9e57caff403381243372a4b360a7 2024-11-25T10:57:15,146 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d6bb9e57caff403381243372a4b360a7, entries=150, sequenceid=139, filesize=11.9 K 2024-11-25T10:57:15,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/2bd83a77e16d43abbb867e23dd1c01c5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/2bd83a77e16d43abbb867e23dd1c01c5 2024-11-25T10:57:15,164 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/2bd83a77e16d43abbb867e23dd1c01c5, entries=150, sequenceid=139, filesize=11.9 K 2024-11-25T10:57:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/bf4fc60eae214a0bb36f370b22385ad6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/bf4fc60eae214a0bb36f370b22385ad6 2024-11-25T10:57:15,176 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/bf4fc60eae214a0bb36f370b22385ad6, entries=150, sequenceid=139, filesize=11.9 K 2024-11-25T10:57:15,177 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e529e4988658b515c2e6b809e2e4ac97 in 600ms, sequenceid=139, compaction requested=false 2024-11-25T10:57:15,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:15,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:15,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-25T10:57:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-25T10:57:15,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-25T10:57:15,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 777 msec 2024-11-25T10:57:15,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 792 msec 2024-11-25T10:57:15,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:57:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:15,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/3c8f1b3b1a0e4b8badff4527dd42664e is 50, key is test_row_0/A:col10/1732532234751/Put/seqid=0 2024-11-25T10:57:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742259_1435 (size=14541) 2024-11-25T10:57:15,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532295500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532295504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532295510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532295522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532295522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T10:57:15,564 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:15,565 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-25T10:57:15,568 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T10:57:15,568 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:15,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532295644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T10:57:15,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532295653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532295665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532295665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532295669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T10:57:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:15,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:15,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T10:57:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T10:57:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:15,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:15,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532295883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532295885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532295885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/3c8f1b3b1a0e4b8badff4527dd42664e 2024-11-25T10:57:15,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532295895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532295914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:15,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/8923d7db77184ca6aaf1648d8403b32d is 50, key is test_row_0/B:col10/1732532234751/Put/seqid=0 2024-11-25T10:57:16,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742260_1436 (size=12151) 2024-11-25T10:57:16,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/8923d7db77184ca6aaf1648d8403b32d 2024-11-25T10:57:16,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e6ac1056de354829916b68c4bf3c008f is 50, key is test_row_0/C:col10/1732532234751/Put/seqid=0 2024-11-25T10:57:16,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T10:57:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742261_1437 (size=12151) 2024-11-25T10:57:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T10:57:16,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T10:57:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532296226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532296227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532296227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532296228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532296245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T10:57:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e6ac1056de354829916b68c4bf3c008f 2024-11-25T10:57:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/3c8f1b3b1a0e4b8badff4527dd42664e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3c8f1b3b1a0e4b8badff4527dd42664e 2024-11-25T10:57:16,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3c8f1b3b1a0e4b8badff4527dd42664e, entries=200, sequenceid=157, filesize=14.2 K 2024-11-25T10:57:16,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/8923d7db77184ca6aaf1648d8403b32d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/8923d7db77184ca6aaf1648d8403b32d 2024-11-25T10:57:16,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/8923d7db77184ca6aaf1648d8403b32d, entries=150, sequenceid=157, filesize=11.9 K 2024-11-25T10:57:16,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e6ac1056de354829916b68c4bf3c008f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6ac1056de354829916b68c4bf3c008f 2024-11-25T10:57:16,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T10:57:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:16,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6ac1056de354829916b68c4bf3c008f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-25T10:57:16,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e529e4988658b515c2e6b809e2e4ac97 in 1123ms, sequenceid=157, compaction requested=true 2024-11-25T10:57:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:57:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:16,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:57:16,556 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:16,556 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:16,564 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:16,564 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:16,564 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,564 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/43e07169859e49d794e0eae16f8f0e1c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d6bb9e57caff403381243372a4b360a7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3c8f1b3b1a0e4b8badff4527dd42664e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=37.9 K 2024-11-25T10:57:16,565 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e07169859e49d794e0eae16f8f0e1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732532231649 2024-11-25T10:57:16,569 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6bb9e57caff403381243372a4b360a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732532233908 2024-11-25T10:57:16,569 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c8f1b3b1a0e4b8badff4527dd42664e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532234748 2024-11-25T10:57:16,572 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:16,572 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:16,573 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,573 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f47e004ceff940189b0244da6ac6fff7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/bf4fc60eae214a0bb36f370b22385ad6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6ac1056de354829916b68c4bf3c008f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=35.6 K 2024-11-25T10:57:16,576 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f47e004ceff940189b0244da6ac6fff7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732532231649 2024-11-25T10:57:16,588 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4fc60eae214a0bb36f370b22385ad6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732532233908 2024-11-25T10:57:16,594 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ac1056de354829916b68c4bf3c008f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532234751 2024-11-25T10:57:16,603 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:16,604 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/fbc3ec79b122417dbd1b272e3e3ae9de is 50, key is test_row_0/A:col10/1732532234751/Put/seqid=0 2024-11-25T10:57:16,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742262_1438 (size=12391) 2024-11-25T10:57:16,663 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:16,664 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/7f21bdbe242e49c4b5996a5be22b3711 is 50, key is test_row_0/C:col10/1732532234751/Put/seqid=0 2024-11-25T10:57:16,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T10:57:16,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,692 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:57:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:16,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T10:57:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cbb3f9543caa4444ae8880eb48f532ad is 50, key is test_row_0/A:col10/1732532235504/Put/seqid=0 2024-11-25T10:57:16,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742263_1439 (size=12391) 2024-11-25T10:57:16,720 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/fbc3ec79b122417dbd1b272e3e3ae9de as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/fbc3ec79b122417dbd1b272e3e3ae9de 2024-11-25T10:57:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:16,752 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/7f21bdbe242e49c4b5996a5be22b3711 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7f21bdbe242e49c4b5996a5be22b3711 2024-11-25T10:57:16,755 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into fbc3ec79b122417dbd1b272e3e3ae9de(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:16,755 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:16,755 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=13, startTime=1732532236553; duration=0sec 2024-11-25T10:57:16,755 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:16,756 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:16,756 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:16,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742264_1440 (size=12151) 2024-11-25T10:57:16,766 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into 7f21bdbe242e49c4b5996a5be22b3711(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:16,766 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cbb3f9543caa4444ae8880eb48f532ad 2024-11-25T10:57:16,766 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:16,766 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=13, startTime=1732532236553; duration=0sec 2024-11-25T10:57:16,767 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:16,767 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:16,773 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:16,773 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:16,773 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:16,773 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5e15828073af4976ae1117d87d02e993, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/2bd83a77e16d43abbb867e23dd1c01c5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/8923d7db77184ca6aaf1648d8403b32d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=35.6 K 2024-11-25T10:57:16,780 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e15828073af4976ae1117d87d02e993, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732532231649 2024-11-25T10:57:16,785 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd83a77e16d43abbb867e23dd1c01c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732532233908 2024-11-25T10:57:16,789 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8923d7db77184ca6aaf1648d8403b32d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532234751 2024-11-25T10:57:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/1d268d61730c4aa9a1ee7d949b860719 is 50, key is test_row_0/B:col10/1732532235504/Put/seqid=0 2024-11-25T10:57:16,820 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:16,821 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/779377e5256f45c287e7edee0ed7230a is 50, key is test_row_0/B:col10/1732532234751/Put/seqid=0 2024-11-25T10:57:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532296813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532296817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532296819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532296821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532296825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742266_1442 (size=12151) 2024-11-25T10:57:16,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742265_1441 (size=12391) 2024-11-25T10:57:16,926 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/779377e5256f45c287e7edee0ed7230a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/779377e5256f45c287e7edee0ed7230a 2024-11-25T10:57:16,937 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into 779377e5256f45c287e7edee0ed7230a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:16,937 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:16,937 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=13, startTime=1732532236553; duration=0sec 2024-11-25T10:57:16,937 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:16,937 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:16,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532296936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532296947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532296947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532296947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:16,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532296952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532297163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532297177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532297179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532297179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532297183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,281 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/1d268d61730c4aa9a1ee7d949b860719 2024-11-25T10:57:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/b712e1d6ae1248baa6311092cafc2448 is 50, key is test_row_0/C:col10/1732532235504/Put/seqid=0 2024-11-25T10:57:17,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742267_1443 (size=12151) 2024-11-25T10:57:17,384 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/b712e1d6ae1248baa6311092cafc2448 2024-11-25T10:57:17,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cbb3f9543caa4444ae8880eb48f532ad as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cbb3f9543caa4444ae8880eb48f532ad 2024-11-25T10:57:17,407 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cbb3f9543caa4444ae8880eb48f532ad, entries=150, sequenceid=174, filesize=11.9 K 2024-11-25T10:57:17,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/1d268d61730c4aa9a1ee7d949b860719 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d268d61730c4aa9a1ee7d949b860719 2024-11-25T10:57:17,412 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d268d61730c4aa9a1ee7d949b860719, entries=150, sequenceid=174, filesize=11.9 K 2024-11-25T10:57:17,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/b712e1d6ae1248baa6311092cafc2448 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/b712e1d6ae1248baa6311092cafc2448 2024-11-25T10:57:17,418 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/b712e1d6ae1248baa6311092cafc2448, entries=150, sequenceid=174, filesize=11.9 K 2024-11-25T10:57:17,425 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for e529e4988658b515c2e6b809e2e4ac97 in 733ms, sequenceid=174, compaction requested=false 2024-11-25T10:57:17,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:17,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-25T10:57:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-25T10:57:17,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-25T10:57:17,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-25T10:57:17,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9050 sec 2024-11-25T10:57:17,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:57:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:17,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:17,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cd734f91168e49c9a924d59b1229f5dc is 50, key is test_row_0/A:col10/1732532237492/Put/seqid=0 2024-11-25T10:57:17,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742268_1444 (size=14541) 2024-11-25T10:57:17,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cd734f91168e49c9a924d59b1229f5dc 2024-11-25T10:57:17,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/cb889b72e2114247ac4bf5413d018e63 is 50, key is test_row_0/B:col10/1732532237492/Put/seqid=0 2024-11-25T10:57:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532297545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742269_1445 (size=12151) 2024-11-25T10:57:17,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/cb889b72e2114247ac4bf5413d018e63 2024-11-25T10:57:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532297603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532297608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532297616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532297622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/cac7c442a6d04200a9f2f0427babce6e is 50, key is test_row_0/C:col10/1732532237492/Put/seqid=0 2024-11-25T10:57:17,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742270_1446 (size=12151) 2024-11-25T10:57:17,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/cac7c442a6d04200a9f2f0427babce6e 2024-11-25T10:57:17,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/cd734f91168e49c9a924d59b1229f5dc as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd734f91168e49c9a924d59b1229f5dc 2024-11-25T10:57:17,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd734f91168e49c9a924d59b1229f5dc, entries=200, sequenceid=195, filesize=14.2 K 2024-11-25T10:57:17,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/cb889b72e2114247ac4bf5413d018e63 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/cb889b72e2114247ac4bf5413d018e63 2024-11-25T10:57:17,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/cb889b72e2114247ac4bf5413d018e63, entries=150, sequenceid=195, filesize=11.9 K 2024-11-25T10:57:17,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/cac7c442a6d04200a9f2f0427babce6e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/cac7c442a6d04200a9f2f0427babce6e 2024-11-25T10:57:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T10:57:17,725 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:17,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/cac7c442a6d04200a9f2f0427babce6e, entries=150, sequenceid=195, filesize=11.9 K 2024-11-25T10:57:17,729 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-25T10:57:17,732 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:17,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:17,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T10:57:17,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e529e4988658b515c2e6b809e2e4ac97 in 245ms, sequenceid=195, compaction requested=true 2024-11-25T10:57:17,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:17,743 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:17,744 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:17,744 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:17,744 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:17,744 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/fbc3ec79b122417dbd1b272e3e3ae9de, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cbb3f9543caa4444ae8880eb48f532ad, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd734f91168e49c9a924d59b1229f5dc] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=38.2 K 2024-11-25T10:57:17,748 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbc3ec79b122417dbd1b272e3e3ae9de, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532234751 2024-11-25T10:57:17,748 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbb3f9543caa4444ae8880eb48f532ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732532235503 2024-11-25T10:57:17,749 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd734f91168e49c9a924d59b1229f5dc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732532236812 2024-11-25T10:57:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:17,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:17,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:17,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:17,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:17,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:17,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:17,754 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:17,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:57:17,755 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:17,755 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:17,755 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:17,755 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/779377e5256f45c287e7edee0ed7230a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d268d61730c4aa9a1ee7d949b860719, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/cb889b72e2114247ac4bf5413d018e63] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=35.8 K 2024-11-25T10:57:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:17,755 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 779377e5256f45c287e7edee0ed7230a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532234751 2024-11-25T10:57:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:17,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:17,755 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d268d61730c4aa9a1ee7d949b860719, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732532235503 2024-11-25T10:57:17,756 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting cb889b72e2114247ac4bf5413d018e63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732532236812 2024-11-25T10:57:17,775 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:17,776 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/d940840e021948dc8fc408c757292a21 is 50, key is test_row_0/A:col10/1732532237492/Put/seqid=0 2024-11-25T10:57:17,779 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:17,779 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/0b6db9c5d2364fda8785547d7ceb2264 is 50, key is test_row_0/B:col10/1732532237492/Put/seqid=0 2024-11-25T10:57:17,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/49049f20bed64b5985cc8839c529307e is 50, key is test_row_0/A:col10/1732532237566/Put/seqid=0 2024-11-25T10:57:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T10:57:17,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742273_1449 (size=14541) 2024-11-25T10:57:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742272_1448 (size=12493) 2024-11-25T10:57:17,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742271_1447 (size=12493) 2024-11-25T10:57:17,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:17,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:17,903 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/d940840e021948dc8fc408c757292a21 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d940840e021948dc8fc408c757292a21 2024-11-25T10:57:17,922 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/0b6db9c5d2364fda8785547d7ceb2264 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/0b6db9c5d2364fda8785547d7ceb2264 2024-11-25T10:57:17,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532297886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,926 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into d940840e021948dc8fc408c757292a21(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:17,926 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:17,926 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=13, startTime=1732532237743; duration=0sec 2024-11-25T10:57:17,926 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:17,926 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:17,926 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532297894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532297896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532297908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532297913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:17,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:17,945 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:17,945 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:17,945 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:17,946 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7f21bdbe242e49c4b5996a5be22b3711, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/b712e1d6ae1248baa6311092cafc2448, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/cac7c442a6d04200a9f2f0427babce6e] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=35.8 K 2024-11-25T10:57:17,950 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f21bdbe242e49c4b5996a5be22b3711, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532234751 2024-11-25T10:57:17,950 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b712e1d6ae1248baa6311092cafc2448, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732532235503 2024-11-25T10:57:17,952 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into 0b6db9c5d2364fda8785547d7ceb2264(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:17,952 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:17,952 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=13, startTime=1732532237753; duration=0sec 2024-11-25T10:57:17,952 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:17,952 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:17,952 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cac7c442a6d04200a9f2f0427babce6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732532236812 2024-11-25T10:57:17,983 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:17,983 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/785d9002dec64cb89cbcbdfe4a5af93e is 50, key is test_row_0/C:col10/1732532237492/Put/seqid=0 2024-11-25T10:57:18,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742274_1450 (size=12493) 2024-11-25T10:57:18,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532298037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532298037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532298037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532298041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T10:57:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532298045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,104 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/785d9002dec64cb89cbcbdfe4a5af93e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/785d9002dec64cb89cbcbdfe4a5af93e 2024-11-25T10:57:18,145 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into 785d9002dec64cb89cbcbdfe4a5af93e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:18,145 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:18,145 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=13, startTime=1732532237753; duration=0sec 2024-11-25T10:57:18,145 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:18,145 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:18,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:18,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/49049f20bed64b5985cc8839c529307e 2024-11-25T10:57:18,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4bde47e1293f4db9bcee75fa91e4c9c5 is 50, key is test_row_0/B:col10/1732532237566/Put/seqid=0 2024-11-25T10:57:18,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532298264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532298264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532298265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532298266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532298266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742275_1451 (size=12151) 2024-11-25T10:57:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T10:57:18,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:18,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:18,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532298586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532298597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532298597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532298597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532298602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:18,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:18,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4bde47e1293f4db9bcee75fa91e4c9c5 2024-11-25T10:57:18,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/d0520bb21f9d4f9e91c1019bf6decbeb is 50, key is test_row_0/C:col10/1732532237566/Put/seqid=0 2024-11-25T10:57:18,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742276_1452 (size=12151) 2024-11-25T10:57:18,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:18,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T10:57:19,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:19,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532299104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:19,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532299117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:19,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:19,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532299123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532299124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532299125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:19,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:19,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:19,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:19,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:19,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/d0520bb21f9d4f9e91c1019bf6decbeb 2024-11-25T10:57:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/49049f20bed64b5985cc8839c529307e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/49049f20bed64b5985cc8839c529307e 2024-11-25T10:57:19,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/49049f20bed64b5985cc8839c529307e, entries=200, sequenceid=210, filesize=14.2 K 2024-11-25T10:57:19,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4bde47e1293f4db9bcee75fa91e4c9c5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4bde47e1293f4db9bcee75fa91e4c9c5 2024-11-25T10:57:19,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4bde47e1293f4db9bcee75fa91e4c9c5, entries=150, sequenceid=210, filesize=11.9 K 2024-11-25T10:57:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/d0520bb21f9d4f9e91c1019bf6decbeb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/d0520bb21f9d4f9e91c1019bf6decbeb 2024-11-25T10:57:19,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/d0520bb21f9d4f9e91c1019bf6decbeb, entries=150, sequenceid=210, filesize=11.9 K 2024-11-25T10:57:19,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e529e4988658b515c2e6b809e2e4ac97 in 1525ms, sequenceid=210, compaction requested=false 2024-11-25T10:57:19,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:19,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T10:57:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:19,332 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:57:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/35a9993d25be4c088beff03a3aa6c4f5 is 50, key is test_row_0/A:col10/1732532237885/Put/seqid=0 2024-11-25T10:57:19,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742277_1453 (size=12151) 2024-11-25T10:57:19,413 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/35a9993d25be4c088beff03a3aa6c4f5 2024-11-25T10:57:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/dd5831f764224779a93466972129293c is 50, key is test_row_0/B:col10/1732532237885/Put/seqid=0 2024-11-25T10:57:19,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742278_1454 (size=12151) 2024-11-25T10:57:19,464 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/dd5831f764224779a93466972129293c 2024-11-25T10:57:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/ab0e417633674cf18cf94a76a82ba498 is 50, key is test_row_0/C:col10/1732532237885/Put/seqid=0 2024-11-25T10:57:19,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742279_1455 (size=12151) 2024-11-25T10:57:19,553 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/ab0e417633674cf18cf94a76a82ba498 2024-11-25T10:57:19,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/35a9993d25be4c088beff03a3aa6c4f5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/35a9993d25be4c088beff03a3aa6c4f5 2024-11-25T10:57:19,608 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/35a9993d25be4c088beff03a3aa6c4f5, entries=150, sequenceid=231, filesize=11.9 K 2024-11-25T10:57:19,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/dd5831f764224779a93466972129293c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/dd5831f764224779a93466972129293c 2024-11-25T10:57:19,622 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/dd5831f764224779a93466972129293c, entries=150, sequenceid=231, filesize=11.9 K 2024-11-25T10:57:19,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/ab0e417633674cf18cf94a76a82ba498 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ab0e417633674cf18cf94a76a82ba498 2024-11-25T10:57:19,639 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ab0e417633674cf18cf94a76a82ba498, entries=150, sequenceid=231, filesize=11.9 K 2024-11-25T10:57:19,640 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for e529e4988658b515c2e6b809e2e4ac97 in 308ms, sequenceid=231, compaction requested=true 2024-11-25T10:57:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-25T10:57:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-25T10:57:19,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-25T10:57:19,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9070 sec 2024-11-25T10:57:19,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.9160 sec 2024-11-25T10:57:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T10:57:19,883 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:19,889 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-25T10:57:19,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T10:57:19,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:19,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T10:57:20,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T10:57:20,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:20,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-25T10:57:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-25T10:57:20,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-25T10:57:20,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 163 msec 2024-11-25T10:57:20,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 169 msec 2024-11-25T10:57:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:20,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T10:57:20,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:20,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:20,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:20,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:20,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:20,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:20,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/f69b61b14a9d48bf8f15f68a5491472a is 50, key is test_row_0/A:col10/1732532240161/Put/seqid=0 2024-11-25T10:57:20,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742280_1456 (size=14541) 2024-11-25T10:57:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T10:57:20,214 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:20,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-25T10:57:20,217 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:20,217 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:20,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:20,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/f69b61b14a9d48bf8f15f68a5491472a 2024-11-25T10:57:20,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/668d8498c5174548b0ebc800a53507bf is 50, key is test_row_0/B:col10/1732532240161/Put/seqid=0 2024-11-25T10:57:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532300256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532240254, max timestamp=1732532240286Num uniques 859; 2024-11-25T10:57:20,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:20,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532240254, max timestamp=1732532240286Num uniques 859; 2024-11-25T10:57:20,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:20,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532240254, max timestamp=1732532240286Num uniques 859; 2024-11-25T10:57:20,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:20,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742281_1457 (size=12151) 2024-11-25T10:57:20,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:20,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532300361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532300386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532300388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532300385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532300400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532300497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532300513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532300514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532300524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:20,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532300624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532300724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532300747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/668d8498c5174548b0ebc800a53507bf 2024-11-25T10:57:20,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532300757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532300758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/270bd7b560a7467a9e7bce5192460740 is 50, key is test_row_0/C:col10/1732532240161/Put/seqid=0 2024-11-25T10:57:20,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:20,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:20,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742282_1458 (size=12151) 2024-11-25T10:57:20,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/270bd7b560a7467a9e7bce5192460740 2024-11-25T10:57:20,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/f69b61b14a9d48bf8f15f68a5491472a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f69b61b14a9d48bf8f15f68a5491472a 2024-11-25T10:57:20,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f69b61b14a9d48bf8f15f68a5491472a, entries=200, sequenceid=243, filesize=14.2 K 2024-11-25T10:57:20,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/668d8498c5174548b0ebc800a53507bf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/668d8498c5174548b0ebc800a53507bf 2024-11-25T10:57:20,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532300945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:20,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/668d8498c5174548b0ebc800a53507bf, entries=150, sequenceid=243, filesize=11.9 K 2024-11-25T10:57:20,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/270bd7b560a7467a9e7bce5192460740 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/270bd7b560a7467a9e7bce5192460740 2024-11-25T10:57:21,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:21,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/270bd7b560a7467a9e7bce5192460740, entries=150, sequenceid=243, filesize=11.9 K 2024-11-25T10:57:21,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e529e4988658b515c2e6b809e2e4ac97 in 884ms, sequenceid=243, compaction requested=true 2024-11-25T10:57:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:57:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:57:21,047 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:21,047 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:21,056 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:21,056 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:21,056 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,057 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/785d9002dec64cb89cbcbdfe4a5af93e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/d0520bb21f9d4f9e91c1019bf6decbeb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ab0e417633674cf18cf94a76a82ba498, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/270bd7b560a7467a9e7bce5192460740] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=47.8 K 2024-11-25T10:57:21,062 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:21,062 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:21,062 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,062 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d940840e021948dc8fc408c757292a21, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/49049f20bed64b5985cc8839c529307e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/35a9993d25be4c088beff03a3aa6c4f5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f69b61b14a9d48bf8f15f68a5491472a] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=52.5 K 2024-11-25T10:57:21,063 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d940840e021948dc8fc408c757292a21, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732532236812 2024-11-25T10:57:21,063 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 785d9002dec64cb89cbcbdfe4a5af93e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732532236812 2024-11-25T10:57:21,067 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49049f20bed64b5985cc8839c529307e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732532237519 2024-11-25T10:57:21,067 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d0520bb21f9d4f9e91c1019bf6decbeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732532237566 2024-11-25T10:57:21,072 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a9993d25be4c088beff03a3aa6c4f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732532237885 2024-11-25T10:57:21,073 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting ab0e417633674cf18cf94a76a82ba498, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732532237885 2024-11-25T10:57:21,073 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69b61b14a9d48bf8f15f68a5491472a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732532240136 2024-11-25T10:57:21,076 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 270bd7b560a7467a9e7bce5192460740, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732532240136 2024-11-25T10:57:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:21,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-25T10:57:21,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:21,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:21,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:21,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:21,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:21,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:21,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/7171a4a488134fdf996759bafaa4814d is 50, key is test_row_0/A:col10/1732532241087/Put/seqid=0 2024-11-25T10:57:21,117 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#388 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:21,117 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/44d8b8d31ac64019b39a86ac9cbd30ad is 50, key is test_row_0/A:col10/1732532240161/Put/seqid=0 2024-11-25T10:57:21,123 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:21,124 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/1adfe44eb01d4349bff9b7e9149608bd is 50, key is test_row_0/C:col10/1732532240161/Put/seqid=0 2024-11-25T10:57:21,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532301129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532301141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532301145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532301151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742283_1459 (size=17181) 2024-11-25T10:57:21,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742284_1460 (size=12629) 2024-11-25T10:57:21,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742285_1461 (size=12629) 2024-11-25T10:57:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532301253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532301253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532301263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532301274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:21,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532301484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532301484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532301484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532301494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532301504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:21,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:21,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:21,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/7171a4a488134fdf996759bafaa4814d 2024-11-25T10:57:21,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/e67723e5ad9042a58e48ca62771b01d9 is 50, key is test_row_0/B:col10/1732532241087/Put/seqid=0 2024-11-25T10:57:21,622 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/44d8b8d31ac64019b39a86ac9cbd30ad as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/44d8b8d31ac64019b39a86ac9cbd30ad 2024-11-25T10:57:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742286_1462 (size=12301) 2024-11-25T10:57:21,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/e67723e5ad9042a58e48ca62771b01d9 2024-11-25T10:57:21,645 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/1adfe44eb01d4349bff9b7e9149608bd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/1adfe44eb01d4349bff9b7e9149608bd 2024-11-25T10:57:21,649 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into 44d8b8d31ac64019b39a86ac9cbd30ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:21,649 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:21,649 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=12, startTime=1732532241047; duration=0sec 2024-11-25T10:57:21,649 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:21,649 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:21,649 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:57:21,664 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:57:21,664 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:21,664 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,664 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/0b6db9c5d2364fda8785547d7ceb2264, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4bde47e1293f4db9bcee75fa91e4c9c5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/dd5831f764224779a93466972129293c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/668d8498c5174548b0ebc800a53507bf] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=47.8 K 2024-11-25T10:57:21,667 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into 1adfe44eb01d4349bff9b7e9149608bd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:21,667 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:21,667 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=12, startTime=1732532241047; duration=0sec 2024-11-25T10:57:21,667 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:21,667 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:21,667 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b6db9c5d2364fda8785547d7ceb2264, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732532236812 2024-11-25T10:57:21,670 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bde47e1293f4db9bcee75fa91e4c9c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732532237566 2024-11-25T10:57:21,672 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5831f764224779a93466972129293c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732532237885 2024-11-25T10:57:21,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/60fffda96613458cb83ebedf81f30a77 is 50, key is test_row_0/C:col10/1732532241087/Put/seqid=0 2024-11-25T10:57:21,677 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 668d8498c5174548b0ebc800a53507bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732532240136 2024-11-25T10:57:21,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:21,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742287_1463 (size=12301) 2024-11-25T10:57:21,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/60fffda96613458cb83ebedf81f30a77 2024-11-25T10:57:21,757 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:21,758 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/6bb5d90d6e7149d3ae82899b629f16cd is 50, key is test_row_0/B:col10/1732532240161/Put/seqid=0 2024-11-25T10:57:21,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/7171a4a488134fdf996759bafaa4814d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/7171a4a488134fdf996759bafaa4814d 2024-11-25T10:57:21,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/7171a4a488134fdf996759bafaa4814d, entries=250, sequenceid=280, filesize=16.8 K 2024-11-25T10:57:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/e67723e5ad9042a58e48ca62771b01d9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e67723e5ad9042a58e48ca62771b01d9 2024-11-25T10:57:21,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e67723e5ad9042a58e48ca62771b01d9, entries=150, sequenceid=280, filesize=12.0 K 2024-11-25T10:57:21,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/60fffda96613458cb83ebedf81f30a77 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/60fffda96613458cb83ebedf81f30a77 2024-11-25T10:57:21,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/60fffda96613458cb83ebedf81f30a77, entries=150, sequenceid=280, filesize=12.0 K 2024-11-25T10:57:21,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for e529e4988658b515c2e6b809e2e4ac97 in 701ms, sequenceid=280, compaction requested=false 2024-11-25T10:57:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:21,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742288_1464 (size=12629) 2024-11-25T10:57:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:21,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:57:21,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:21,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:21,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:21,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:21,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:21,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:21,815 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/6bb5d90d6e7149d3ae82899b629f16cd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6bb5d90d6e7149d3ae82899b629f16cd 2024-11-25T10:57:21,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/6e6c39436da94a788897dfcedb9277bd is 50, key is test_row_0/A:col10/1732532241813/Put/seqid=0 2024-11-25T10:57:21,830 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into 6bb5d90d6e7149d3ae82899b629f16cd(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:21,830 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:21,830 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=12, startTime=1732532241047; duration=0sec 2024-11-25T10:57:21,830 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:21,830 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:21,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:21,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:21,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:21,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:21,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742289_1465 (size=14741) 2024-11-25T10:57:21,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/6e6c39436da94a788897dfcedb9277bd 2024-11-25T10:57:21,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/27a584104431480bb9ce6248bbceb985 is 50, key is test_row_0/B:col10/1732532241813/Put/seqid=0 2024-11-25T10:57:21,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532241887, max timestamp=1732532241955Num uniques 859; 2024-11-25T10:57:21,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:21,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532241887, max timestamp=1732532241955Num uniques 859; 2024-11-25T10:57:21,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:21,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532241887, max timestamp=1732532241955Num uniques 859; 2024-11-25T10:57:21,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:21,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742290_1466 (size=12301) 2024-11-25T10:57:22,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/27a584104431480bb9ce6248bbceb985 2024-11-25T10:57:22,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:22,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:22,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/9e671a136d3f45d681ff09c32718de88 is 50, key is test_row_0/C:col10/1732532241813/Put/seqid=0 2024-11-25T10:57:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532302055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532302057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742291_1467 (size=12301) 2024-11-25T10:57:22,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/9e671a136d3f45d681ff09c32718de88 2024-11-25T10:57:22,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532302082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532302089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/6e6c39436da94a788897dfcedb9277bd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/6e6c39436da94a788897dfcedb9277bd 2024-11-25T10:57:22,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/6e6c39436da94a788897dfcedb9277bd, entries=200, sequenceid=292, filesize=14.4 K 2024-11-25T10:57:22,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/27a584104431480bb9ce6248bbceb985 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/27a584104431480bb9ce6248bbceb985 2024-11-25T10:57:22,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/27a584104431480bb9ce6248bbceb985, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T10:57:22,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/9e671a136d3f45d681ff09c32718de88 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9e671a136d3f45d681ff09c32718de88 2024-11-25T10:57:22,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9e671a136d3f45d681ff09c32718de88, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T10:57:22,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e529e4988658b515c2e6b809e2e4ac97 in 385ms, sequenceid=292, compaction requested=true 2024-11-25T10:57:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:22,199 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:22,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:57:22,199 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:22,208 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:22,208 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:22,208 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,208 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/44d8b8d31ac64019b39a86ac9cbd30ad, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/7171a4a488134fdf996759bafaa4814d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/6e6c39436da94a788897dfcedb9277bd] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=43.5 K 2024-11-25T10:57:22,211 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:22,211 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d8b8d31ac64019b39a86ac9cbd30ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732532240136 2024-11-25T10:57:22,211 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:22,211 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,211 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6bb5d90d6e7149d3ae82899b629f16cd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e67723e5ad9042a58e48ca62771b01d9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/27a584104431480bb9ce6248bbceb985] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.4 K 2024-11-25T10:57:22,212 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7171a4a488134fdf996759bafaa4814d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732532240346 2024-11-25T10:57:22,212 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb5d90d6e7149d3ae82899b629f16cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732532240136 2024-11-25T10:57:22,212 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6c39436da94a788897dfcedb9277bd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732532241124 2024-11-25T10:57:22,212 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e67723e5ad9042a58e48ca62771b01d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732532240386 2024-11-25T10:57:22,213 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 27a584104431480bb9ce6248bbceb985, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732532241124 2024-11-25T10:57:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:22,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-25T10:57:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T10:57:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T10:57:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T10:57:22,222 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:22,223 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/c1f9dfe9e814437a82bb3de078807e58 is 50, key is test_row_0/A:col10/1732532241813/Put/seqid=0 2024-11-25T10:57:22,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/560fc6bd87e34028b36eb711d812f323 is 50, key is test_row_0/A:col10/1732532242087/Put/seqid=0 2024-11-25T10:57:22,231 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:22,231 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/69a009dff28c469dbaaeff6385514030 is 50, key is test_row_0/B:col10/1732532241813/Put/seqid=0 2024-11-25T10:57:22,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742293_1469 (size=17181) 2024-11-25T10:57:22,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742292_1468 (size=12881) 2024-11-25T10:57:22,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/560fc6bd87e34028b36eb711d812f323 2024-11-25T10:57:22,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742294_1470 (size=12881) 2024-11-25T10:57:22,258 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/69a009dff28c469dbaaeff6385514030 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/69a009dff28c469dbaaeff6385514030 2024-11-25T10:57:22,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/3e752c45cf284750ab454c56c505b45c is 50, key is test_row_0/B:col10/1732532242087/Put/seqid=0 2024-11-25T10:57:22,268 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into 69a009dff28c469dbaaeff6385514030(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:22,268 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:22,268 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=13, startTime=1732532242199; duration=0sec 2024-11-25T10:57:22,268 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:22,268 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:22,268 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:22,269 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:22,269 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:22,269 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,270 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/1adfe44eb01d4349bff9b7e9149608bd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/60fffda96613458cb83ebedf81f30a77, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9e671a136d3f45d681ff09c32718de88] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.4 K 2024-11-25T10:57:22,270 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1adfe44eb01d4349bff9b7e9149608bd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732532240136 2024-11-25T10:57:22,270 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 60fffda96613458cb83ebedf81f30a77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732532240386 2024-11-25T10:57:22,271 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e671a136d3f45d681ff09c32718de88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732532241124 2024-11-25T10:57:22,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532302247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532302249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,283 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:22,284 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/7e454a809be14ca2a92d85378326f774 is 50, key is test_row_0/C:col10/1732532241813/Put/seqid=0 2024-11-25T10:57:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532302251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532302275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742295_1471 (size=12301) 2024-11-25T10:57:22,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/3e752c45cf284750ab454c56c505b45c 2024-11-25T10:57:22,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/2a337fed576346219e390c5bbdbb0d6f is 50, key is test_row_0/C:col10/1732532242087/Put/seqid=0 2024-11-25T10:57:22,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742296_1472 (size=12881) 2024-11-25T10:57:22,344 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/7e454a809be14ca2a92d85378326f774 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7e454a809be14ca2a92d85378326f774 2024-11-25T10:57:22,349 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into 7e454a809be14ca2a92d85378326f774(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:22,349 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:22,349 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=13, startTime=1732532242199; duration=0sec 2024-11-25T10:57:22,349 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:22,349 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:22,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742297_1473 (size=12301) 2024-11-25T10:57:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532302384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532302386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532302393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532302403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532302533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,563 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in e529e4988658b515c2e6b809e2e4ac97 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532302603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532302603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:22,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:22,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532302624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532302624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,656 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/c1f9dfe9e814437a82bb3de078807e58 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c1f9dfe9e814437a82bb3de078807e58 2024-11-25T10:57:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:22,677 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into c1f9dfe9e814437a82bb3de078807e58(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:22,677 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:22,677 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=13, startTime=1732532242199; duration=0sec 2024-11-25T10:57:22,677 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:22,677 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:22,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/2a337fed576346219e390c5bbdbb0d6f 2024-11-25T10:57:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/560fc6bd87e34028b36eb711d812f323 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/560fc6bd87e34028b36eb711d812f323 2024-11-25T10:57:22,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/560fc6bd87e34028b36eb711d812f323, entries=250, sequenceid=332, filesize=16.8 K 2024-11-25T10:57:22,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/3e752c45cf284750ab454c56c505b45c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3e752c45cf284750ab454c56c505b45c 2024-11-25T10:57:22,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3e752c45cf284750ab454c56c505b45c, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T10:57:22,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/2a337fed576346219e390c5bbdbb0d6f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2a337fed576346219e390c5bbdbb0d6f 2024-11-25T10:57:22,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2a337fed576346219e390c5bbdbb0d6f, entries=150, sequenceid=332, filesize=12.0 K 2024-11-25T10:57:22,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for e529e4988658b515c2e6b809e2e4ac97 in 680ms, sequenceid=332, compaction requested=false 2024-11-25T10:57:22,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:22,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:57:22,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:22,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:22,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:22,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:22,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:22,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:22,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/70cf15343f814ecdaa0bc2a06b4f87e2 is 50, key is test_row_0/A:col10/1732532242939/Put/seqid=0 2024-11-25T10:57:22,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:22,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:22,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742298_1474 (size=17181) 2024-11-25T10:57:23,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/70cf15343f814ecdaa0bc2a06b4f87e2 2024-11-25T10:57:23,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/3dfbc4852e9d40ce95e37ce26bf23a43 is 50, key is test_row_0/B:col10/1732532242939/Put/seqid=0 2024-11-25T10:57:23,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532303086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532303090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532303103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532303108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:23,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:23,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742299_1475 (size=12301) 2024-11-25T10:57:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532303214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532303223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532303244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532303244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532303435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:23,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:23,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532303445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532303477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532303477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/3dfbc4852e9d40ce95e37ce26bf23a43 2024-11-25T10:57:23,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e77443da37194a4b94cbe9465c6cd94d is 50, key is test_row_0/C:col10/1732532242939/Put/seqid=0 2024-11-25T10:57:23,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:23,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742300_1476 (size=12301) 2024-11-25T10:57:23,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:23,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:23,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532303764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532303787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532303805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532303805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:23,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:23,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:23,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:23,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:24,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e77443da37194a4b94cbe9465c6cd94d 2024-11-25T10:57:24,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/70cf15343f814ecdaa0bc2a06b4f87e2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/70cf15343f814ecdaa0bc2a06b4f87e2 2024-11-25T10:57:24,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/70cf15343f814ecdaa0bc2a06b4f87e2, entries=250, sequenceid=345, filesize=16.8 K 2024-11-25T10:57:24,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:24,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:24,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:24,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/3dfbc4852e9d40ce95e37ce26bf23a43 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3dfbc4852e9d40ce95e37ce26bf23a43 2024-11-25T10:57:24,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3dfbc4852e9d40ce95e37ce26bf23a43, entries=150, sequenceid=345, filesize=12.0 K 2024-11-25T10:57:24,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e77443da37194a4b94cbe9465c6cd94d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e77443da37194a4b94cbe9465c6cd94d 2024-11-25T10:57:24,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e77443da37194a4b94cbe9465c6cd94d, entries=150, sequenceid=345, filesize=12.0 K 2024-11-25T10:57:24,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for e529e4988658b515c2e6b809e2e4ac97 in 1190ms, sequenceid=345, compaction requested=true 2024-11-25T10:57:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:57:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T10:57:24,137 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:24,137 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:24,141 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:24,141 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:24,141 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:24,142 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7e454a809be14ca2a92d85378326f774, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2a337fed576346219e390c5bbdbb0d6f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e77443da37194a4b94cbe9465c6cd94d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.6 K 2024-11-25T10:57:24,142 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e454a809be14ca2a92d85378326f774, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732532241124 2024-11-25T10:57:24,144 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a337fed576346219e390c5bbdbb0d6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732532242077 2024-11-25T10:57:24,145 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e77443da37194a4b94cbe9465c6cd94d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732532242273 2024-11-25T10:57:24,152 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:24,152 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:24,152 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:24,152 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c1f9dfe9e814437a82bb3de078807e58, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/560fc6bd87e34028b36eb711d812f323, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/70cf15343f814ecdaa0bc2a06b4f87e2] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=46.1 K 2024-11-25T10:57:24,152 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f9dfe9e814437a82bb3de078807e58, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732532241124 2024-11-25T10:57:24,157 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 560fc6bd87e34028b36eb711d812f323, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732532242055 2024-11-25T10:57:24,158 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 70cf15343f814ecdaa0bc2a06b4f87e2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732532242247 2024-11-25T10:57:24,171 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:24,171 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/70afceb55eb145db8fb6ab5efde5b227 is 50, key is test_row_0/C:col10/1732532242939/Put/seqid=0 2024-11-25T10:57:24,211 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:24,212 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/c206e23780d048f8a8781b6f0cae6d12 is 50, key is test_row_0/A:col10/1732532242939/Put/seqid=0 2024-11-25T10:57:24,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T10:57:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:24,238 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-25T10:57:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:24,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742301_1477 (size=12983) 2024-11-25T10:57:24,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742302_1478 (size=12983) 2024-11-25T10:57:24,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/f241f5946495463c9dc2863c4f271dfe is 50, key is test_row_0/A:col10/1732532243069/Put/seqid=0 2024-11-25T10:57:24,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:24,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742303_1479 (size=12301) 2024-11-25T10:57:24,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532304344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532304371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532304373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532304377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532304487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532304505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532304505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532304509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532304576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,600 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in e529e4988658b515c2e6b809e2e4ac97 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4344 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,653 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/70afceb55eb145db8fb6ab5efde5b227 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/70afceb55eb145db8fb6ab5efde5b227 2024-11-25T10:57:24,658 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into 70afceb55eb145db8fb6ab5efde5b227(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:24,658 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:24,658 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=13, startTime=1732532244136; duration=0sec 2024-11-25T10:57:24,658 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:24,658 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:24,658 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:24,668 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:24,669 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:24,669 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:24,669 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/69a009dff28c469dbaaeff6385514030, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3e752c45cf284750ab454c56c505b45c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3dfbc4852e9d40ce95e37ce26bf23a43] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.6 K 2024-11-25T10:57:24,671 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a009dff28c469dbaaeff6385514030, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732532241124 2024-11-25T10:57:24,671 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e752c45cf284750ab454c56c505b45c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732532242077 2024-11-25T10:57:24,672 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dfbc4852e9d40ce95e37ce26bf23a43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732532242273 2024-11-25T10:57:24,675 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/c206e23780d048f8a8781b6f0cae6d12 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c206e23780d048f8a8781b6f0cae6d12 2024-11-25T10:57:24,680 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into c206e23780d048f8a8781b6f0cae6d12(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:24,680 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:24,681 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=13, startTime=1732532244136; duration=0sec 2024-11-25T10:57:24,681 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:24,681 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:24,707 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:24,708 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/f2a8cba7cb654d529d7b40d87a666f51 is 50, key is test_row_0/B:col10/1732532242939/Put/seqid=0 2024-11-25T10:57:24,732 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/f241f5946495463c9dc2863c4f271dfe 2024-11-25T10:57:24,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532304713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742304_1480 (size=12983) 2024-11-25T10:57:24,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532304734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532304734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532304734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:24,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/b58dac17d9524b69942890eba8bb2d86 is 50, key is test_row_0/B:col10/1732532243069/Put/seqid=0 2024-11-25T10:57:24,767 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/f2a8cba7cb654d529d7b40d87a666f51 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/f2a8cba7cb654d529d7b40d87a666f51 2024-11-25T10:57:24,772 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into f2a8cba7cb654d529d7b40d87a666f51(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:24,772 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:24,772 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=13, startTime=1732532244136; duration=0sec 2024-11-25T10:57:24,772 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:24,772 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:24,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742305_1481 (size=12301) 2024-11-25T10:57:24,807 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/b58dac17d9524b69942890eba8bb2d86 2024-11-25T10:57:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e6586173aaa744399f144c96c6728e63 is 50, key is test_row_0/C:col10/1732532243069/Put/seqid=0 2024-11-25T10:57:24,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742306_1482 (size=12301) 2024-11-25T10:57:24,884 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e6586173aaa744399f144c96c6728e63 2024-11-25T10:57:24,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/f241f5946495463c9dc2863c4f271dfe as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f241f5946495463c9dc2863c4f271dfe 2024-11-25T10:57:24,945 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f241f5946495463c9dc2863c4f271dfe, entries=150, sequenceid=367, filesize=12.0 K 2024-11-25T10:57:24,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/b58dac17d9524b69942890eba8bb2d86 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/b58dac17d9524b69942890eba8bb2d86 2024-11-25T10:57:24,997 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/b58dac17d9524b69942890eba8bb2d86, entries=150, sequenceid=367, filesize=12.0 K 2024-11-25T10:57:25,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e6586173aaa744399f144c96c6728e63 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6586173aaa744399f144c96c6728e63 2024-11-25T10:57:25,043 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6586173aaa744399f144c96c6728e63, entries=150, sequenceid=367, filesize=12.0 K 2024-11-25T10:57:25,044 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for e529e4988658b515c2e6b809e2e4ac97 in 806ms, sequenceid=367, compaction requested=false 2024-11-25T10:57:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-25T10:57:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-25T10:57:25,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-25T10:57:25,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.8310 sec 2024-11-25T10:57:25,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 4.8350 sec 2024-11-25T10:57:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:25,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T10:57:25,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:25,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:25,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:25,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:25,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:25,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:25,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/3a40e5eb29ff4137a5c2e08656e0e945 is 50, key is test_row_0/A:col10/1732532244368/Put/seqid=0 2024-11-25T10:57:25,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742307_1483 (size=14741) 2024-11-25T10:57:25,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/3a40e5eb29ff4137a5c2e08656e0e945 2024-11-25T10:57:25,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/d594c92430ae4f5792fa9a49889659bb is 50, key is test_row_0/B:col10/1732532244368/Put/seqid=0 2024-11-25T10:57:25,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532305190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532305196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532305211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532305218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742308_1484 (size=12301) 2024-11-25T10:57:25,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/d594c92430ae4f5792fa9a49889659bb 2024-11-25T10:57:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532305334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532305344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532305344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532305353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/00820c2c40744608925c6657939c0ba5 is 50, key is test_row_0/C:col10/1732532244368/Put/seqid=0 2024-11-25T10:57:25,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742309_1485 (size=12301) 2024-11-25T10:57:25,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/00820c2c40744608925c6657939c0ba5 2024-11-25T10:57:25,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/3a40e5eb29ff4137a5c2e08656e0e945 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3a40e5eb29ff4137a5c2e08656e0e945 2024-11-25T10:57:25,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3a40e5eb29ff4137a5c2e08656e0e945, entries=200, sequenceid=382, filesize=14.4 K 2024-11-25T10:57:25,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/d594c92430ae4f5792fa9a49889659bb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/d594c92430ae4f5792fa9a49889659bb 2024-11-25T10:57:25,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532305573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532305576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532305584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532305584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:25,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/d594c92430ae4f5792fa9a49889659bb, entries=150, sequenceid=382, filesize=12.0 K 2024-11-25T10:57:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/00820c2c40744608925c6657939c0ba5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/00820c2c40744608925c6657939c0ba5 2024-11-25T10:57:25,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/00820c2c40744608925c6657939c0ba5, entries=150, sequenceid=382, filesize=12.0 K 2024-11-25T10:57:25,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e529e4988658b515c2e6b809e2e4ac97 in 608ms, sequenceid=382, compaction requested=true 2024-11-25T10:57:25,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:25,681 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:25,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:25,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:25,682 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:25,682 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:25,682 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:25,682 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:25,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:25,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:25,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:25,682 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c206e23780d048f8a8781b6f0cae6d12, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f241f5946495463c9dc2863c4f271dfe, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3a40e5eb29ff4137a5c2e08656e0e945] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=39.1 K 2024-11-25T10:57:25,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:25,689 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c206e23780d048f8a8781b6f0cae6d12, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732532242273 2024-11-25T10:57:25,693 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:25,693 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:25,693 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:25,693 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/f2a8cba7cb654d529d7b40d87a666f51, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/b58dac17d9524b69942890eba8bb2d86, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/d594c92430ae4f5792fa9a49889659bb] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.7 K 2024-11-25T10:57:25,693 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f241f5946495463c9dc2863c4f271dfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732532243069 2024-11-25T10:57:25,697 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a8cba7cb654d529d7b40d87a666f51, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732532242273 2024-11-25T10:57:25,697 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a40e5eb29ff4137a5c2e08656e0e945, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732532244342 2024-11-25T10:57:25,701 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting b58dac17d9524b69942890eba8bb2d86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732532243069 2024-11-25T10:57:25,702 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d594c92430ae4f5792fa9a49889659bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732532244342 2024-11-25T10:57:25,743 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:25,744 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/dc8d8520e46446379d866d383384c535 is 50, key is test_row_0/A:col10/1732532244368/Put/seqid=0 2024-11-25T10:57:25,746 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:25,747 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/c3615e367efb4ff7bc270faf9f921009 is 50, key is test_row_0/B:col10/1732532244368/Put/seqid=0 2024-11-25T10:57:25,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742310_1486 (size=13085) 2024-11-25T10:57:25,831 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/dc8d8520e46446379d866d383384c535 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/dc8d8520e46446379d866d383384c535 2024-11-25T10:57:25,838 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into dc8d8520e46446379d866d383384c535(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:25,838 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:25,838 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=13, startTime=1732532245681; duration=0sec 2024-11-25T10:57:25,839 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:25,839 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:25,839 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:25,840 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:25,840 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:25,840 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:25,840 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/70afceb55eb145db8fb6ab5efde5b227, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6586173aaa744399f144c96c6728e63, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/00820c2c40744608925c6657939c0ba5] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.7 K 2024-11-25T10:57:25,840 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70afceb55eb145db8fb6ab5efde5b227, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732532242273 2024-11-25T10:57:25,841 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6586173aaa744399f144c96c6728e63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732532243069 2024-11-25T10:57:25,841 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00820c2c40744608925c6657939c0ba5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732532244342 2024-11-25T10:57:25,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742311_1487 (size=13085) 2024-11-25T10:57:25,867 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:25,867 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/fbfa187c8f784b39bae89874c573f29e is 50, key is test_row_0/C:col10/1732532244368/Put/seqid=0 2024-11-25T10:57:25,885 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/c3615e367efb4ff7bc270faf9f921009 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c3615e367efb4ff7bc270faf9f921009 2024-11-25T10:57:25,890 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into c3615e367efb4ff7bc270faf9f921009(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:25,890 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:25,890 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=13, startTime=1732532245682; duration=0sec 2024-11-25T10:57:25,890 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:25,890 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:25,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742312_1488 (size=13085) 2024-11-25T10:57:25,929 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/fbfa187c8f784b39bae89874c573f29e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fbfa187c8f784b39bae89874c573f29e 2024-11-25T10:57:25,937 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into fbfa187c8f784b39bae89874c573f29e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:25,937 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:25,937 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=13, startTime=1732532245682; duration=0sec 2024-11-25T10:57:25,937 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:25,937 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:25,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532245156, max timestamp=1732532245938Num uniques 859; 2024-11-25T10:57:25,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:25,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532245156, max timestamp=1732532245938Num uniques 859; 2024-11-25T10:57:25,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532245156, max timestamp=1732532245938Num uniques 859; 2024-11-25T10:57:25,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:25,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:25,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-25T10:57:25,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:25,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/62b6aa219d2c4940a26c122ca5dedbac is 50, key is test_row_0/A:col10/1732532245212/Put/seqid=0 2024-11-25T10:57:26,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742313_1489 (size=14741) 2024-11-25T10:57:26,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732532246046, max timestamp=1732532246130Num uniques 859; 2024-11-25T10:57:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T10:57:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732532246046, max timestamp=1732532246130Num uniques 859; 2024-11-25T10:57:26,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T10:57:26,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732532246046, max timestamp=1732532246130Num uniques 859; 2024-11-25T10:57:26,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T10:57:26,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532306217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532306229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532306253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532306255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532306364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532306364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532306373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532306373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/62b6aa219d2c4940a26c122ca5dedbac 2024-11-25T10:57:26,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/c7ba31badcff4c44978916261cd70461 is 50, key is test_row_0/B:col10/1732532245212/Put/seqid=0 2024-11-25T10:57:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742314_1490 (size=12301) 2024-11-25T10:57:26,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532306595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532306595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532306614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532306614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/c7ba31badcff4c44978916261cd70461 2024-11-25T10:57:26,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/48594bedb4eb4fd18ba322ef5aab078c is 50, key is test_row_0/C:col10/1732532245212/Put/seqid=0 2024-11-25T10:57:26,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532306944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532306944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532306944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532306945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742315_1491 (size=12301) 2024-11-25T10:57:27,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/48594bedb4eb4fd18ba322ef5aab078c 2024-11-25T10:57:27,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/62b6aa219d2c4940a26c122ca5dedbac as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/62b6aa219d2c4940a26c122ca5dedbac 2024-11-25T10:57:27,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/62b6aa219d2c4940a26c122ca5dedbac, entries=200, sequenceid=410, filesize=14.4 K 2024-11-25T10:57:27,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/c7ba31badcff4c44978916261cd70461 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c7ba31badcff4c44978916261cd70461 2024-11-25T10:57:27,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c7ba31badcff4c44978916261cd70461, entries=150, sequenceid=410, filesize=12.0 K 2024-11-25T10:57:27,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/48594bedb4eb4fd18ba322ef5aab078c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/48594bedb4eb4fd18ba322ef5aab078c 2024-11-25T10:57:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532307474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532307474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532307475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:27,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532307475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:27,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/48594bedb4eb4fd18ba322ef5aab078c, entries=150, sequenceid=410, filesize=12.0 K 2024-11-25T10:57:27,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for e529e4988658b515c2e6b809e2e4ac97 in 1538ms, sequenceid=410, compaction requested=false 2024-11-25T10:57:27,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T10:57:28,405 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:28,406 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-25T10:57:28,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:28,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:28,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T10:57:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:28,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-25T10:57:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:28,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T10:57:28,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/4abdb4a0c2cc458ba0d6d8bd17fcac37 is 50, key is test_row_0/A:col10/1732532246202/Put/seqid=0 2024-11-25T10:57:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532308541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532308542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:28,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:28,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532308547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532308549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742316_1492 (size=14741) 2024-11-25T10:57:28,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38178 deadline: 1732532308625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,657 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in e529e4988658b515c2e6b809e2e4ac97 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8400 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532308674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532308674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532308684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532308684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:28,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:28,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:28,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:28,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T10:57:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:28,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:28,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:28,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:28,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532308894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532308897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532308914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532308914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:28,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/4abdb4a0c2cc458ba0d6d8bd17fcac37 2024-11-25T10:57:29,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/bb15df26404145a489932b03b3f0a248 is 50, key is test_row_0/B:col10/1732532246202/Put/seqid=0 2024-11-25T10:57:29,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T10:57:29,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742317_1493 (size=12301) 2024-11-25T10:57:29,130 INFO [Thread-1857 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1857. 2024-11-25T10:57:29,130 DEBUG [Thread-1857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:29,130 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,130 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:29,135 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-11-25T10:57:29,135 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:29,135 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:29,136 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,138 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-25T10:57:29,138 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:29,138 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,138 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:29,144 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-25T10:57:29,144 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:29,144 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,144 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,144 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:29,152 INFO [Thread-1855 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1855. 2024-11-25T10:57:29,152 DEBUG [Thread-1855 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:29,153 DEBUG [Thread-1855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,153 DEBUG [Thread-1855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:29,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532309203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532309214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532309223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532309223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/bb15df26404145a489932b03b3f0a248 2024-11-25T10:57:29,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/2490fb74366d4343843ff7e538373a87 is 50, key is test_row_0/C:col10/1732532246202/Put/seqid=0 2024-11-25T10:57:29,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742318_1494 (size=12301) 2024-11-25T10:57:29,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T10:57:29,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38206 deadline: 1732532309714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38162 deadline: 1732532309724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38140 deadline: 1732532309733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38136 deadline: 1732532309733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 , the old value is region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., hostname=6bfc37bad115,44671,1732532110409, seqNum=2 because the exception is null or not the one we care about 2024-11-25T10:57:29,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:29,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/2490fb74366d4343843ff7e538373a87 2024-11-25T10:57:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/4abdb4a0c2cc458ba0d6d8bd17fcac37 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/4abdb4a0c2cc458ba0d6d8bd17fcac37 2024-11-25T10:57:29,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/4abdb4a0c2cc458ba0d6d8bd17fcac37, entries=200, sequenceid=446, filesize=14.4 K 2024-11-25T10:57:29,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/bb15df26404145a489932b03b3f0a248 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/bb15df26404145a489932b03b3f0a248 2024-11-25T10:57:29,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/bb15df26404145a489932b03b3f0a248, entries=150, sequenceid=446, filesize=12.0 K 2024-11-25T10:57:29,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/2490fb74366d4343843ff7e538373a87 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2490fb74366d4343843ff7e538373a87 2024-11-25T10:57:29,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2490fb74366d4343843ff7e538373a87, entries=150, sequenceid=446, filesize=12.0 K 2024-11-25T10:57:29,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for e529e4988658b515c2e6b809e2e4ac97 in 1378ms, sequenceid=446, compaction requested=true 2024-11-25T10:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:29,892 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:29,892 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e529e4988658b515c2e6b809e2e4ac97:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:29,893 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:29,893 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:29,893 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/B is initiating minor compaction (all files) 2024-11-25T10:57:29,893 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/A is initiating minor compaction (all files) 2024-11-25T10:57:29,893 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/A in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,893 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/B in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,893 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/dc8d8520e46446379d866d383384c535, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/62b6aa219d2c4940a26c122ca5dedbac, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/4abdb4a0c2cc458ba0d6d8bd17fcac37] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=41.6 K 2024-11-25T10:57:29,893 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c3615e367efb4ff7bc270faf9f921009, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c7ba31badcff4c44978916261cd70461, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/bb15df26404145a489932b03b3f0a248] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.8 K 2024-11-25T10:57:29,894 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c3615e367efb4ff7bc270faf9f921009, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732532244342 2024-11-25T10:57:29,894 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc8d8520e46446379d866d383384c535, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732532244342 2024-11-25T10:57:29,894 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b6aa219d2c4940a26c122ca5dedbac, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732532245212 2024-11-25T10:57:29,894 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ba31badcff4c44978916261cd70461, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732532245212 2024-11-25T10:57:29,895 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4abdb4a0c2cc458ba0d6d8bd17fcac37, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732532246202 2024-11-25T10:57:29,895 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bb15df26404145a489932b03b3f0a248, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732532246202 2024-11-25T10:57:29,904 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:29,905 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/a01de926d67e401da5a5e62941502d1e is 50, key is test_row_0/A:col10/1732532246202/Put/seqid=0 2024-11-25T10:57:29,911 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:29,911 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4f23ab8ce2d74b0cbe9cd6077bce2203 is 50, key is test_row_0/B:col10/1732532246202/Put/seqid=0 2024-11-25T10:57:29,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742320_1496 (size=13187) 2024-11-25T10:57:29,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742319_1495 (size=13187) 2024-11-25T10:57:29,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T10:57:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:29,971 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:57:29,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:29,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:29,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:29,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:29,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:29,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/65eca4ed924745218338688e5e517fd1 is 50, key is test_row_0/A:col10/1732532248540/Put/seqid=0 2024-11-25T10:57:29,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742321_1497 (size=12301) 2024-11-25T10:57:30,324 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4f23ab8ce2d74b0cbe9cd6077bce2203 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4f23ab8ce2d74b0cbe9cd6077bce2203 2024-11-25T10:57:30,324 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/a01de926d67e401da5a5e62941502d1e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/a01de926d67e401da5a5e62941502d1e 2024-11-25T10:57:30,329 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/B of e529e4988658b515c2e6b809e2e4ac97 into 4f23ab8ce2d74b0cbe9cd6077bce2203(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:30,329 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/A of e529e4988658b515c2e6b809e2e4ac97 into a01de926d67e401da5a5e62941502d1e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:30,329 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:30,329 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:30,329 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/A, priority=13, startTime=1732532249892; duration=0sec 2024-11-25T10:57:30,329 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/B, priority=13, startTime=1732532249892; duration=0sec 2024-11-25T10:57:30,329 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:30,329 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:B 2024-11-25T10:57:30,329 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:30,329 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:A 2024-11-25T10:57:30,329 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:30,330 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:30,330 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): e529e4988658b515c2e6b809e2e4ac97/C is initiating minor compaction (all files) 2024-11-25T10:57:30,330 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e529e4988658b515c2e6b809e2e4ac97/C in TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:30,330 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fbfa187c8f784b39bae89874c573f29e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/48594bedb4eb4fd18ba322ef5aab078c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2490fb74366d4343843ff7e538373a87] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp, totalSize=36.8 K 2024-11-25T10:57:30,330 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting fbfa187c8f784b39bae89874c573f29e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732532244342 2024-11-25T10:57:30,330 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 48594bedb4eb4fd18ba322ef5aab078c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732532245212 2024-11-25T10:57:30,331 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2490fb74366d4343843ff7e538373a87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732532246202 2024-11-25T10:57:30,337 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e529e4988658b515c2e6b809e2e4ac97#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:30,338 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/4f12b9ff21e845fdb466fdbc434f8180 is 50, key is test_row_0/C:col10/1732532246202/Put/seqid=0 2024-11-25T10:57:30,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742322_1498 (size=13187) 2024-11-25T10:57:30,390 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/65eca4ed924745218338688e5e517fd1 2024-11-25T10:57:30,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4b22bd28cb9842e4a0a97ac345561067 is 50, key is test_row_0/B:col10/1732532248540/Put/seqid=0 2024-11-25T10:57:30,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742323_1499 (size=12301) 2024-11-25T10:57:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T10:57:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:30,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. as already flushing 2024-11-25T10:57:30,726 INFO [Thread-1848 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1848. 2024-11-25T10:57:30,726 DEBUG [Thread-1848 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:30,726 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,726 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:30,735 INFO [Thread-1850 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1850. 2024-11-25T10:57:30,735 DEBUG [Thread-1850 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:30,735 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,736 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:30,746 INFO [Thread-1844 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1844. 2024-11-25T10:57:30,746 DEBUG [Thread-1844 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:30,746 DEBUG [Thread-1844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,746 DEBUG [Thread-1844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:30,748 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/4f12b9ff21e845fdb466fdbc434f8180 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4f12b9ff21e845fdb466fdbc434f8180 2024-11-25T10:57:30,754 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e529e4988658b515c2e6b809e2e4ac97/C of e529e4988658b515c2e6b809e2e4ac97 into 4f12b9ff21e845fdb466fdbc434f8180(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:30,754 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:30,754 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., storeName=e529e4988658b515c2e6b809e2e4ac97/C, priority=13, startTime=1732532249892; duration=0sec 2024-11-25T10:57:30,754 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:30,754 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e529e4988658b515c2e6b809e2e4ac97:C 2024-11-25T10:57:30,761 INFO [Thread-1846 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1846. 2024-11-25T10:57:30,761 DEBUG [Thread-1846 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:30,761 DEBUG [Thread-1846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,761 DEBUG [Thread-1846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:30,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:30,832 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4b22bd28cb9842e4a0a97ac345561067 2024-11-25T10:57:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/55544207823a4a6896262ef15b37887a is 50, key is test_row_0/C:col10/1732532248540/Put/seqid=0 2024-11-25T10:57:30,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742324_1500 (size=12301) 2024-11-25T10:57:31,254 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/55544207823a4a6896262ef15b37887a 2024-11-25T10:57:31,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/65eca4ed924745218338688e5e517fd1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/65eca4ed924745218338688e5e517fd1 2024-11-25T10:57:31,262 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/65eca4ed924745218338688e5e517fd1, entries=150, sequenceid=456, filesize=12.0 K 2024-11-25T10:57:31,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/4b22bd28cb9842e4a0a97ac345561067 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4b22bd28cb9842e4a0a97ac345561067 2024-11-25T10:57:31,266 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4b22bd28cb9842e4a0a97ac345561067, entries=150, sequenceid=456, filesize=12.0 K 2024-11-25T10:57:31,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/55544207823a4a6896262ef15b37887a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/55544207823a4a6896262ef15b37887a 2024-11-25T10:57:31,270 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/55544207823a4a6896262ef15b37887a, entries=150, sequenceid=456, filesize=12.0 K 2024-11-25T10:57:31,271 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for e529e4988658b515c2e6b809e2e4ac97 in 1300ms, sequenceid=456, compaction requested=false 2024-11-25T10:57:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e529e4988658b515c2e6b809e2e4ac97: 2024-11-25T10:57:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-25T10:57:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-25T10:57:31,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-25T10:57:31,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8580 sec 2024-11-25T10:57:31,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.8680 sec 2024-11-25T10:57:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T10:57:32,584 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:37,468 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T10:57:38,715 INFO [Thread-1852 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1852. 2024-11-25T10:57:38,715 DEBUG [Thread-1852 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:57:38,715 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:38,716 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:38,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 653 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1955 rows 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 642 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1926 rows 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 645 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1935 rows 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 654 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1962 rows 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 650 2024-11-25T10:57:38,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1950 rows 2024-11-25T10:57:38,716 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:57:38,716 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:57:38,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:38,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:38,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:57:38,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T10:57:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T10:57:38,722 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532258721"}]},"ts":"1732532258721"} 2024-11-25T10:57:38,723 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T10:57:38,723 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T10:57:38,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:57:38,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, UNASSIGN}] 2024-11-25T10:57:38,734 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, UNASSIGN 2024-11-25T10:57:38,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=e529e4988658b515c2e6b809e2e4ac97, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:38,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, UNASSIGN because future has completed 2024-11-25T10:57:38,752 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:57:38,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:57:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T10:57:38,905 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:38,905 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:57:38,905 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing e529e4988658b515c2e6b809e2e4ac97, disabling compactions & flushes 2024-11-25T10:57:38,905 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:38,905 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:38,905 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. after waiting 0 ms 2024-11-25T10:57:38,905 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:38,905 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing e529e4988658b515c2e6b809e2e4ac97 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T10:57:38,906 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=A 2024-11-25T10:57:38,906 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:38,906 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=B 2024-11-25T10:57:38,906 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:38,906 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e529e4988658b515c2e6b809e2e4ac97, store=C 2024-11-25T10:57:38,906 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:38,910 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/801582898aae457b90972dd113f0cf0c is 50, key is test_row_1/A:col10/1732532250754/Put/seqid=0 2024-11-25T10:57:38,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742325_1501 (size=9857) 2024-11-25T10:57:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T10:57:39,316 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/801582898aae457b90972dd113f0cf0c 2024-11-25T10:57:39,342 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/fd000b86d73547b6afba7782bc0d354c is 50, key is test_row_1/B:col10/1732532250754/Put/seqid=0 2024-11-25T10:57:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T10:57:39,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742326_1502 (size=9857) 2024-11-25T10:57:39,769 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/fd000b86d73547b6afba7782bc0d354c 2024-11-25T10:57:39,777 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e9245e2dc4d84b508b83ab41d506034b is 50, key is test_row_1/C:col10/1732532250754/Put/seqid=0 2024-11-25T10:57:39,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742327_1503 (size=9857) 2024-11-25T10:57:39,817 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e9245e2dc4d84b508b83ab41d506034b 2024-11-25T10:57:39,844 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/A/801582898aae457b90972dd113f0cf0c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/801582898aae457b90972dd113f0cf0c 2024-11-25T10:57:39,847 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/801582898aae457b90972dd113f0cf0c, entries=100, sequenceid=467, filesize=9.6 K 2024-11-25T10:57:39,848 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/B/fd000b86d73547b6afba7782bc0d354c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/fd000b86d73547b6afba7782bc0d354c 2024-11-25T10:57:39,852 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/fd000b86d73547b6afba7782bc0d354c, entries=100, sequenceid=467, filesize=9.6 K 2024-11-25T10:57:39,852 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/.tmp/C/e9245e2dc4d84b508b83ab41d506034b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e9245e2dc4d84b508b83ab41d506034b 2024-11-25T10:57:39,855 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e9245e2dc4d84b508b83ab41d506034b, entries=100, sequenceid=467, filesize=9.6 K 2024-11-25T10:57:39,856 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e529e4988658b515c2e6b809e2e4ac97 in 951ms, sequenceid=467, compaction requested=true 2024-11-25T10:57:39,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/8d5d8725d0444a969ed4778a594e0ba2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/40e0372647934e16b4e7df15c463bbc1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/ca6eddbc813844feb351372e34868d2f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/001efef2d2fe4dd7a7a23b50da5d4fa7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/43e07169859e49d794e0eae16f8f0e1c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d6bb9e57caff403381243372a4b360a7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3c8f1b3b1a0e4b8badff4527dd42664e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/fbc3ec79b122417dbd1b272e3e3ae9de, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cbb3f9543caa4444ae8880eb48f532ad, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd734f91168e49c9a924d59b1229f5dc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d940840e021948dc8fc408c757292a21, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/49049f20bed64b5985cc8839c529307e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/35a9993d25be4c088beff03a3aa6c4f5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f69b61b14a9d48bf8f15f68a5491472a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/44d8b8d31ac64019b39a86ac9cbd30ad, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/7171a4a488134fdf996759bafaa4814d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/6e6c39436da94a788897dfcedb9277bd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c1f9dfe9e814437a82bb3de078807e58, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/560fc6bd87e34028b36eb711d812f323, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/70cf15343f814ecdaa0bc2a06b4f87e2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c206e23780d048f8a8781b6f0cae6d12, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f241f5946495463c9dc2863c4f271dfe, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3a40e5eb29ff4137a5c2e08656e0e945, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/dc8d8520e46446379d866d383384c535, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/62b6aa219d2c4940a26c122ca5dedbac, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/4abdb4a0c2cc458ba0d6d8bd17fcac37] to archive 2024-11-25T10:57:39,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:39,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/8d5d8725d0444a969ed4778a594e0ba2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/8d5d8725d0444a969ed4778a594e0ba2 2024-11-25T10:57:39,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/40e0372647934e16b4e7df15c463bbc1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/40e0372647934e16b4e7df15c463bbc1 2024-11-25T10:57:39,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/ca6eddbc813844feb351372e34868d2f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/ca6eddbc813844feb351372e34868d2f 2024-11-25T10:57:39,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/001efef2d2fe4dd7a7a23b50da5d4fa7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/001efef2d2fe4dd7a7a23b50da5d4fa7 2024-11-25T10:57:39,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/43e07169859e49d794e0eae16f8f0e1c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/43e07169859e49d794e0eae16f8f0e1c 2024-11-25T10:57:39,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d6bb9e57caff403381243372a4b360a7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d6bb9e57caff403381243372a4b360a7 2024-11-25T10:57:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T10:57:39,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3c8f1b3b1a0e4b8badff4527dd42664e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3c8f1b3b1a0e4b8badff4527dd42664e 2024-11-25T10:57:39,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/fbc3ec79b122417dbd1b272e3e3ae9de to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/fbc3ec79b122417dbd1b272e3e3ae9de 2024-11-25T10:57:39,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cbb3f9543caa4444ae8880eb48f532ad to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cbb3f9543caa4444ae8880eb48f532ad 2024-11-25T10:57:39,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd734f91168e49c9a924d59b1229f5dc to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/cd734f91168e49c9a924d59b1229f5dc 2024-11-25T10:57:39,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d940840e021948dc8fc408c757292a21 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/d940840e021948dc8fc408c757292a21 2024-11-25T10:57:39,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/49049f20bed64b5985cc8839c529307e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/49049f20bed64b5985cc8839c529307e 2024-11-25T10:57:39,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/35a9993d25be4c088beff03a3aa6c4f5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/35a9993d25be4c088beff03a3aa6c4f5 2024-11-25T10:57:39,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f69b61b14a9d48bf8f15f68a5491472a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f69b61b14a9d48bf8f15f68a5491472a 2024-11-25T10:57:39,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/44d8b8d31ac64019b39a86ac9cbd30ad to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/44d8b8d31ac64019b39a86ac9cbd30ad 2024-11-25T10:57:39,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/7171a4a488134fdf996759bafaa4814d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/7171a4a488134fdf996759bafaa4814d 2024-11-25T10:57:39,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/6e6c39436da94a788897dfcedb9277bd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/6e6c39436da94a788897dfcedb9277bd 2024-11-25T10:57:39,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c1f9dfe9e814437a82bb3de078807e58 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c1f9dfe9e814437a82bb3de078807e58 2024-11-25T10:57:39,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/560fc6bd87e34028b36eb711d812f323 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/560fc6bd87e34028b36eb711d812f323 2024-11-25T10:57:39,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/70cf15343f814ecdaa0bc2a06b4f87e2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/70cf15343f814ecdaa0bc2a06b4f87e2 2024-11-25T10:57:39,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c206e23780d048f8a8781b6f0cae6d12 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/c206e23780d048f8a8781b6f0cae6d12 2024-11-25T10:57:39,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f241f5946495463c9dc2863c4f271dfe to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/f241f5946495463c9dc2863c4f271dfe 2024-11-25T10:57:39,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3a40e5eb29ff4137a5c2e08656e0e945 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/3a40e5eb29ff4137a5c2e08656e0e945 2024-11-25T10:57:39,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/dc8d8520e46446379d866d383384c535 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/dc8d8520e46446379d866d383384c535 2024-11-25T10:57:39,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/62b6aa219d2c4940a26c122ca5dedbac to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/62b6aa219d2c4940a26c122ca5dedbac 2024-11-25T10:57:39,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/4abdb4a0c2cc458ba0d6d8bd17fcac37 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/4abdb4a0c2cc458ba0d6d8bd17fcac37 2024-11-25T10:57:39,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/145cfc7641d04636aa407f4b35b1a2ac, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ecee5b4d9754c85a3d25f03ab49f8de, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d801460a439467aafbe31feb7776be2, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5e15828073af4976ae1117d87d02e993, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ba11db194714b2b85ef753b1b0e0b4b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/2bd83a77e16d43abbb867e23dd1c01c5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/779377e5256f45c287e7edee0ed7230a, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/8923d7db77184ca6aaf1648d8403b32d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d268d61730c4aa9a1ee7d949b860719, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/0b6db9c5d2364fda8785547d7ceb2264, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/cb889b72e2114247ac4bf5413d018e63, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4bde47e1293f4db9bcee75fa91e4c9c5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/dd5831f764224779a93466972129293c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6bb5d90d6e7149d3ae82899b629f16cd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/668d8498c5174548b0ebc800a53507bf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e67723e5ad9042a58e48ca62771b01d9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/69a009dff28c469dbaaeff6385514030, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/27a584104431480bb9ce6248bbceb985, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3e752c45cf284750ab454c56c505b45c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/f2a8cba7cb654d529d7b40d87a666f51, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3dfbc4852e9d40ce95e37ce26bf23a43, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/b58dac17d9524b69942890eba8bb2d86, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c3615e367efb4ff7bc270faf9f921009, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/d594c92430ae4f5792fa9a49889659bb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c7ba31badcff4c44978916261cd70461, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/bb15df26404145a489932b03b3f0a248] to archive 2024-11-25T10:57:39,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:39,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/145cfc7641d04636aa407f4b35b1a2ac to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/145cfc7641d04636aa407f4b35b1a2ac 2024-11-25T10:57:39,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ecee5b4d9754c85a3d25f03ab49f8de to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ecee5b4d9754c85a3d25f03ab49f8de 2024-11-25T10:57:39,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d801460a439467aafbe31feb7776be2 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d801460a439467aafbe31feb7776be2 2024-11-25T10:57:39,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5e15828073af4976ae1117d87d02e993 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5e15828073af4976ae1117d87d02e993 2024-11-25T10:57:39,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ba11db194714b2b85ef753b1b0e0b4b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/5ba11db194714b2b85ef753b1b0e0b4b 2024-11-25T10:57:39,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/2bd83a77e16d43abbb867e23dd1c01c5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/2bd83a77e16d43abbb867e23dd1c01c5 2024-11-25T10:57:39,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/779377e5256f45c287e7edee0ed7230a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/779377e5256f45c287e7edee0ed7230a 2024-11-25T10:57:39,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/8923d7db77184ca6aaf1648d8403b32d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/8923d7db77184ca6aaf1648d8403b32d 2024-11-25T10:57:39,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d268d61730c4aa9a1ee7d949b860719 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/1d268d61730c4aa9a1ee7d949b860719 2024-11-25T10:57:39,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/0b6db9c5d2364fda8785547d7ceb2264 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/0b6db9c5d2364fda8785547d7ceb2264 2024-11-25T10:57:39,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/cb889b72e2114247ac4bf5413d018e63 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/cb889b72e2114247ac4bf5413d018e63 2024-11-25T10:57:39,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4bde47e1293f4db9bcee75fa91e4c9c5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4bde47e1293f4db9bcee75fa91e4c9c5 2024-11-25T10:57:39,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/dd5831f764224779a93466972129293c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/dd5831f764224779a93466972129293c 2024-11-25T10:57:39,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6bb5d90d6e7149d3ae82899b629f16cd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/6bb5d90d6e7149d3ae82899b629f16cd 2024-11-25T10:57:39,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/668d8498c5174548b0ebc800a53507bf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/668d8498c5174548b0ebc800a53507bf 2024-11-25T10:57:39,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e67723e5ad9042a58e48ca62771b01d9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/e67723e5ad9042a58e48ca62771b01d9 2024-11-25T10:57:39,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/69a009dff28c469dbaaeff6385514030 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/69a009dff28c469dbaaeff6385514030 2024-11-25T10:57:39,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/27a584104431480bb9ce6248bbceb985 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/27a584104431480bb9ce6248bbceb985 2024-11-25T10:57:39,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3e752c45cf284750ab454c56c505b45c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3e752c45cf284750ab454c56c505b45c 2024-11-25T10:57:39,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/f2a8cba7cb654d529d7b40d87a666f51 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/f2a8cba7cb654d529d7b40d87a666f51 2024-11-25T10:57:39,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3dfbc4852e9d40ce95e37ce26bf23a43 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/3dfbc4852e9d40ce95e37ce26bf23a43 2024-11-25T10:57:39,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/b58dac17d9524b69942890eba8bb2d86 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/b58dac17d9524b69942890eba8bb2d86 2024-11-25T10:57:39,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c3615e367efb4ff7bc270faf9f921009 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c3615e367efb4ff7bc270faf9f921009 2024-11-25T10:57:39,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/d594c92430ae4f5792fa9a49889659bb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/d594c92430ae4f5792fa9a49889659bb 2024-11-25T10:57:39,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c7ba31badcff4c44978916261cd70461 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/c7ba31badcff4c44978916261cd70461 2024-11-25T10:57:39,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/bb15df26404145a489932b03b3f0a248 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/bb15df26404145a489932b03b3f0a248 2024-11-25T10:57:39,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ed83e57ec5604cf6b84957a3f73b78aa, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/5a9c06c1d7954766964d18e31b44b2f9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f88857dd61cc495da27127dc62b07a20, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f47e004ceff940189b0244da6ac6fff7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9b6b00ae0bdf4e93b707f8eb23a9b466, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/bf4fc60eae214a0bb36f370b22385ad6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7f21bdbe242e49c4b5996a5be22b3711, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6ac1056de354829916b68c4bf3c008f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/b712e1d6ae1248baa6311092cafc2448, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/785d9002dec64cb89cbcbdfe4a5af93e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/cac7c442a6d04200a9f2f0427babce6e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/d0520bb21f9d4f9e91c1019bf6decbeb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ab0e417633674cf18cf94a76a82ba498, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/1adfe44eb01d4349bff9b7e9149608bd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/270bd7b560a7467a9e7bce5192460740, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/60fffda96613458cb83ebedf81f30a77, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7e454a809be14ca2a92d85378326f774, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9e671a136d3f45d681ff09c32718de88, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2a337fed576346219e390c5bbdbb0d6f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/70afceb55eb145db8fb6ab5efde5b227, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e77443da37194a4b94cbe9465c6cd94d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6586173aaa744399f144c96c6728e63, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fbfa187c8f784b39bae89874c573f29e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/00820c2c40744608925c6657939c0ba5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/48594bedb4eb4fd18ba322ef5aab078c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2490fb74366d4343843ff7e538373a87] to archive 2024-11-25T10:57:39,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:57:39,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ed83e57ec5604cf6b84957a3f73b78aa to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ed83e57ec5604cf6b84957a3f73b78aa 2024-11-25T10:57:39,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/5a9c06c1d7954766964d18e31b44b2f9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/5a9c06c1d7954766964d18e31b44b2f9 2024-11-25T10:57:39,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f88857dd61cc495da27127dc62b07a20 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f88857dd61cc495da27127dc62b07a20 2024-11-25T10:57:39,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f47e004ceff940189b0244da6ac6fff7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/f47e004ceff940189b0244da6ac6fff7 2024-11-25T10:57:39,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9b6b00ae0bdf4e93b707f8eb23a9b466 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9b6b00ae0bdf4e93b707f8eb23a9b466 2024-11-25T10:57:39,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/bf4fc60eae214a0bb36f370b22385ad6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/bf4fc60eae214a0bb36f370b22385ad6 2024-11-25T10:57:39,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7f21bdbe242e49c4b5996a5be22b3711 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7f21bdbe242e49c4b5996a5be22b3711 2024-11-25T10:57:39,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6ac1056de354829916b68c4bf3c008f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6ac1056de354829916b68c4bf3c008f 2024-11-25T10:57:39,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/b712e1d6ae1248baa6311092cafc2448 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/b712e1d6ae1248baa6311092cafc2448 2024-11-25T10:57:39,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/785d9002dec64cb89cbcbdfe4a5af93e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/785d9002dec64cb89cbcbdfe4a5af93e 2024-11-25T10:57:39,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/cac7c442a6d04200a9f2f0427babce6e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/cac7c442a6d04200a9f2f0427babce6e 2024-11-25T10:57:39,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/d0520bb21f9d4f9e91c1019bf6decbeb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/d0520bb21f9d4f9e91c1019bf6decbeb 2024-11-25T10:57:39,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ab0e417633674cf18cf94a76a82ba498 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/ab0e417633674cf18cf94a76a82ba498 2024-11-25T10:57:39,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/1adfe44eb01d4349bff9b7e9149608bd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/1adfe44eb01d4349bff9b7e9149608bd 2024-11-25T10:57:39,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/270bd7b560a7467a9e7bce5192460740 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/270bd7b560a7467a9e7bce5192460740 2024-11-25T10:57:39,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/60fffda96613458cb83ebedf81f30a77 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/60fffda96613458cb83ebedf81f30a77 2024-11-25T10:57:39,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7e454a809be14ca2a92d85378326f774 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/7e454a809be14ca2a92d85378326f774 2024-11-25T10:57:39,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9e671a136d3f45d681ff09c32718de88 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/9e671a136d3f45d681ff09c32718de88 2024-11-25T10:57:39,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2a337fed576346219e390c5bbdbb0d6f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2a337fed576346219e390c5bbdbb0d6f 2024-11-25T10:57:39,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/70afceb55eb145db8fb6ab5efde5b227 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/70afceb55eb145db8fb6ab5efde5b227 2024-11-25T10:57:39,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e77443da37194a4b94cbe9465c6cd94d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e77443da37194a4b94cbe9465c6cd94d 2024-11-25T10:57:39,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6586173aaa744399f144c96c6728e63 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e6586173aaa744399f144c96c6728e63 2024-11-25T10:57:39,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fbfa187c8f784b39bae89874c573f29e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/fbfa187c8f784b39bae89874c573f29e 2024-11-25T10:57:39,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/00820c2c40744608925c6657939c0ba5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/00820c2c40744608925c6657939c0ba5 2024-11-25T10:57:39,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/48594bedb4eb4fd18ba322ef5aab078c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/48594bedb4eb4fd18ba322ef5aab078c 2024-11-25T10:57:39,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2490fb74366d4343843ff7e538373a87 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/2490fb74366d4343843ff7e538373a87 2024-11-25T10:57:40,026 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/recovered.edits/470.seqid, newMaxSeqId=470, maxSeqId=1 2024-11-25T10:57:40,028 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. 2024-11-25T10:57:40,028 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for e529e4988658b515c2e6b809e2e4ac97: Waiting for close lock at 1732532258905Running coprocessor pre-close hooks at 1732532258905Disabling compacts and flushes for region at 1732532258905Disabling writes for close at 1732532258905Obtaining lock to block concurrent updates at 1732532258905Preparing flush snapshotting stores in e529e4988658b515c2e6b809e2e4ac97 at 1732532258905Finished memstore snapshotting TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732532258906 (+1 ms)Flushing stores of TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97. at 1732532258906Flushing e529e4988658b515c2e6b809e2e4ac97/A: creating writer at 1732532258907 (+1 ms)Flushing e529e4988658b515c2e6b809e2e4ac97/A: appending metadata at 1732532258909 (+2 ms)Flushing e529e4988658b515c2e6b809e2e4ac97/A: closing flushed file at 1732532258909Flushing e529e4988658b515c2e6b809e2e4ac97/B: creating writer at 1732532259328 (+419 ms)Flushing e529e4988658b515c2e6b809e2e4ac97/B: appending metadata at 1732532259341 (+13 ms)Flushing e529e4988658b515c2e6b809e2e4ac97/B: closing flushed file at 1732532259341Flushing e529e4988658b515c2e6b809e2e4ac97/C: creating writer at 1732532259773 (+432 ms)Flushing e529e4988658b515c2e6b809e2e4ac97/C: appending metadata at 1732532259776 (+3 ms)Flushing e529e4988658b515c2e6b809e2e4ac97/C: closing flushed file at 1732532259776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c935323: reopening flushed file at 1732532259843 (+67 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f99f0e: reopening flushed file at 1732532259847 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@462b92de: reopening flushed file at 1732532259852 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e529e4988658b515c2e6b809e2e4ac97 in 951ms, sequenceid=467, compaction requested=true at 1732532259856 (+4 ms)Writing region close event to WAL at 1732532259988 (+132 ms)Running coprocessor post-close hooks at 1732532260028 (+40 ms)Closed at 1732532260028 2024-11-25T10:57:40,033 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:40,034 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=e529e4988658b515c2e6b809e2e4ac97, regionState=CLOSED 2024-11-25T10:57:40,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:57:40,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-25T10:57:40,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure e529e4988658b515c2e6b809e2e4ac97, server=6bfc37bad115,44671,1732532110409 in 1.2930 sec 2024-11-25T10:57:40,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-25T10:57:40,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e529e4988658b515c2e6b809e2e4ac97, UNASSIGN in 1.3150 sec 2024-11-25T10:57:40,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-25T10:57:40,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3320 sec 2024-11-25T10:57:40,059 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532260059"}]},"ts":"1732532260059"} 2024-11-25T10:57:40,061 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T10:57:40,061 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T10:57:40,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3440 sec 2024-11-25T10:57:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T10:57:40,874 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:40,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T10:57:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:40,876 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T10:57:40,877 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:40,879 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:40,881 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/recovered.edits] 2024-11-25T10:57:40,884 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/65eca4ed924745218338688e5e517fd1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/65eca4ed924745218338688e5e517fd1 2024-11-25T10:57:40,886 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/801582898aae457b90972dd113f0cf0c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/801582898aae457b90972dd113f0cf0c 2024-11-25T10:57:40,887 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/a01de926d67e401da5a5e62941502d1e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/A/a01de926d67e401da5a5e62941502d1e 2024-11-25T10:57:40,889 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4b22bd28cb9842e4a0a97ac345561067 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4b22bd28cb9842e4a0a97ac345561067 2024-11-25T10:57:40,891 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4f23ab8ce2d74b0cbe9cd6077bce2203 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/4f23ab8ce2d74b0cbe9cd6077bce2203 2024-11-25T10:57:40,892 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/fd000b86d73547b6afba7782bc0d354c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/B/fd000b86d73547b6afba7782bc0d354c 2024-11-25T10:57:40,894 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4f12b9ff21e845fdb466fdbc434f8180 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/4f12b9ff21e845fdb466fdbc434f8180 2024-11-25T10:57:40,895 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/55544207823a4a6896262ef15b37887a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/55544207823a4a6896262ef15b37887a 2024-11-25T10:57:40,896 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e9245e2dc4d84b508b83ab41d506034b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/C/e9245e2dc4d84b508b83ab41d506034b 2024-11-25T10:57:40,898 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/recovered.edits/470.seqid to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97/recovered.edits/470.seqid 2024-11-25T10:57:40,898 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/e529e4988658b515c2e6b809e2e4ac97 2024-11-25T10:57:40,898 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T10:57:40,900 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:40,903 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T10:57:40,905 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T10:57:40,906 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:40,906 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T10:57:40,906 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732532260906"}]},"ts":"9223372036854775807"} 2024-11-25T10:57:40,908 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T10:57:40,908 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e529e4988658b515c2e6b809e2e4ac97, NAME => 'TestAcidGuarantees,,1732532227802.e529e4988658b515c2e6b809e2e4ac97.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T10:57:40,908 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T10:57:40,909 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732532260908"}]},"ts":"9223372036854775807"} 2024-11-25T10:57:40,910 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T10:57:40,911 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:40,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-25T10:57:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-25T10:57:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T10:57:40,984 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:40,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 251), OpenFileDescriptor=443 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1232 (was 1401), ProcessCount=11 (was 11), AvailableMemoryMB=2143 (was 688) - AvailableMemoryMB LEAK? - 2024-11-25T10:57:41,003 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=1232, ProcessCount=11, AvailableMemoryMB=2143 2024-11-25T10:57:41,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:57:41,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:57:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:41,006 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T10:57:41,006 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:41,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-25T10:57:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T10:57:41,007 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T10:57:41,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742328_1504 (size=1061) 2024-11-25T10:57:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T10:57:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T10:57:41,414 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770 2024-11-25T10:57:41,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742329_1505 (size=53) 2024-11-25T10:57:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T10:57:41,819 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:57:41,819 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fb047475327d71fbf830ed8c3260c99d, disabling compactions & flushes 2024-11-25T10:57:41,819 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:41,819 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:41,819 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. after waiting 0 ms 2024-11-25T10:57:41,819 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:41,819 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:41,819 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb047475327d71fbf830ed8c3260c99d: Waiting for close lock at 1732532261819Disabling compacts and flushes for region at 1732532261819Disabling writes for close at 1732532261819Writing region close event to WAL at 1732532261819Closed at 1732532261819 2024-11-25T10:57:41,821 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T10:57:41,821 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732532261821"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532261821"}]},"ts":"1732532261821"} 2024-11-25T10:57:41,823 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T10:57:41,823 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T10:57:41,824 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532261824"}]},"ts":"1732532261824"} 2024-11-25T10:57:41,825 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T10:57:41,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, ASSIGN}] 2024-11-25T10:57:41,826 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, ASSIGN 2024-11-25T10:57:41,827 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, ASSIGN; state=OFFLINE, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=false 2024-11-25T10:57:41,978 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:41,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, ASSIGN because future has completed 2024-11-25T10:57:41,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:57:42,134 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:42,135 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:57:42,135 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,135 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:57:42,135 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,135 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,136 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,137 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:42,137 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb047475327d71fbf830ed8c3260c99d columnFamilyName A 2024-11-25T10:57:42,137 DEBUG [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:42,138 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(327): Store=fb047475327d71fbf830ed8c3260c99d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:42,138 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,139 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:42,139 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb047475327d71fbf830ed8c3260c99d columnFamilyName B 2024-11-25T10:57:42,139 DEBUG [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:42,139 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(327): Store=fb047475327d71fbf830ed8c3260c99d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:42,139 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,140 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:42,140 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb047475327d71fbf830ed8c3260c99d columnFamilyName C 2024-11-25T10:57:42,140 DEBUG [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:42,141 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(327): Store=fb047475327d71fbf830ed8c3260c99d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:42,141 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:42,141 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,141 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,142 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,142 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,142 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,143 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:57:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T10:57:42,144 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,145 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T10:57:42,146 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened fb047475327d71fbf830ed8c3260c99d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63500117, jitterRate=-0.05377452075481415}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:57:42,146 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:42,146 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for fb047475327d71fbf830ed8c3260c99d: Running coprocessor pre-open hook at 1732532262135Writing region info on filesystem at 1732532262135Initializing all the Stores at 1732532262136 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532262136Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532262136Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532262136Cleaning up temporary data from old regions at 1732532262142 (+6 ms)Running coprocessor post-open hooks at 1732532262146 (+4 ms)Region opened successfully at 1732532262146 2024-11-25T10:57:42,147 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., pid=134, masterSystemTime=1732532262132 2024-11-25T10:57:42,148 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:42,148 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:42,149 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:42,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:57:42,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-25T10:57:42,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 in 171 msec 2024-11-25T10:57:42,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-25T10:57:42,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, ASSIGN in 327 msec 2024-11-25T10:57:42,154 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T10:57:42,155 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532262154"}]},"ts":"1732532262154"} 2024-11-25T10:57:42,156 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T10:57:42,156 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T10:57:42,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-25T10:57:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-25T10:57:43,153 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:43,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,155 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f195736, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e2838f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,158 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,160 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T10:57:43,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39013,1732532109078 2024-11-25T10:57:43,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f374ecc 2024-11-25T10:57:43,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T10:57:43,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T10:57:43,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T10:57:43,163 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T10:57:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T10:57:43,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742330_1506 (size=1097) 2024-11-25T10:57:43,177 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T10:57:43,177 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T10:57:43,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:57:43,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, REOPEN/MOVE}] 2024-11-25T10:57:43,182 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, REOPEN/MOVE 2024-11-25T10:57:43,182 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, REOPEN/MOVE because future has completed 2024-11-25T10:57:43,184 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:57:43,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:57:43,336 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,336 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:57:43,336 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing fb047475327d71fbf830ed8c3260c99d, disabling compactions & flushes 2024-11-25T10:57:43,336 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,336 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,336 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. after waiting 0 ms 2024-11-25T10:57:43,336 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,340 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T10:57:43,340 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,340 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for fb047475327d71fbf830ed8c3260c99d: Waiting for close lock at 1732532263336Running coprocessor pre-close hooks at 1732532263336Disabling compacts and flushes for region at 1732532263336Disabling writes for close at 1732532263336Writing region close event to WAL at 1732532263337 (+1 ms)Running coprocessor post-close hooks at 1732532263340 (+3 ms)Closed at 1732532263340 2024-11-25T10:57:43,340 WARN [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: fb047475327d71fbf830ed8c3260c99d to self. 2024-11-25T10:57:43,342 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,342 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=CLOSED 2024-11-25T10:57:43,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:57:43,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-25T10:57:43,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 in 160 msec 2024-11-25T10:57:43,346 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, REOPEN/MOVE; state=CLOSED, location=6bfc37bad115,44671,1732532110409; forceNewPlan=false, retain=true 2024-11-25T10:57:43,497 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=OPENING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, REOPEN/MOVE because future has completed 2024-11-25T10:57:43,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:57:43,653 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,653 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T10:57:43,653 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,653 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T10:57:43,653 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,653 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,654 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,655 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:43,655 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb047475327d71fbf830ed8c3260c99d columnFamilyName A 2024-11-25T10:57:43,656 DEBUG [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:43,657 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(327): Store=fb047475327d71fbf830ed8c3260c99d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:43,657 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,657 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:43,657 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb047475327d71fbf830ed8c3260c99d columnFamilyName B 2024-11-25T10:57:43,657 DEBUG [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:43,658 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(327): Store=fb047475327d71fbf830ed8c3260c99d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:43,658 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,658 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T10:57:43,658 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb047475327d71fbf830ed8c3260c99d columnFamilyName C 2024-11-25T10:57:43,658 DEBUG [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:43,659 INFO [StoreOpener-fb047475327d71fbf830ed8c3260c99d-1 {}] regionserver.HStore(327): Store=fb047475327d71fbf830ed8c3260c99d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T10:57:43,659 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,659 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,659 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,660 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,661 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,661 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,661 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T10:57:43,662 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,663 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened fb047475327d71fbf830ed8c3260c99d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69397002, jitterRate=0.03409591317176819}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T10:57:43,663 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,663 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for fb047475327d71fbf830ed8c3260c99d: Running coprocessor pre-open hook at 1732532263653Writing region info on filesystem at 1732532263653Initializing all the Stores at 1732532263654 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532263654Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532263654Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532263654Cleaning up temporary data from old regions at 1732532263661 (+7 ms)Running coprocessor post-open hooks at 1732532263663 (+2 ms)Region opened successfully at 1732532263663 2024-11-25T10:57:43,664 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., pid=139, masterSystemTime=1732532263650 2024-11-25T10:57:43,665 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,665 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,666 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=OPEN, openSeqNum=5, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:57:43,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-25T10:57:43,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 in 170 msec 2024-11-25T10:57:43,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-25T10:57:43,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, REOPEN/MOVE in 489 msec 2024-11-25T10:57:43,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-25T10:57:43,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-25T10:57:43,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 511 msec 2024-11-25T10:57:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-25T10:57:43,675 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:43,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e2e00d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a6ac94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,680 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28667fba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,681 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ef78a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74550928, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269a17ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f741a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72118659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,688 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20bcfe45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2628dda5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccb4a3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b8c761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,694 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c70fbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219b9470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a017155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,698 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129d419f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,699 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1534b3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,700 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6d17f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39013,-1 for getting cluster id 2024-11-25T10:57:43,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T10:57:43,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd15501a3-200b-4410-9329-faccae53cbbc' 2024-11-25T10:57:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T10:57:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d15501a3-200b-4410-9329-faccae53cbbc" 2024-11-25T10:57:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ffd2df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39013,-1] 2024-11-25T10:57:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T10:57:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:57:43,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T10:57:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e60eaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T10:57:43,704 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,704 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,704 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,704 DEBUG [Thread-2274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,704 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,705 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,705 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,706 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:43,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,707 DEBUG [Thread-2272 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-25T10:57:43,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,709 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T10:57:43,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,712 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T10:57:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,44671,1732532110409, seqNum=-1] 2024-11-25T10:57:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T10:57:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T10:57:43,715 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5] 2024-11-25T10:57:43,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:43,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:43,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:57:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:43,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f1f7873d966448f1a566535187e923b0_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532263718/Put/seqid=0 2024-11-25T10:57:43,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742331_1507 (size=12154) 2024-11-25T10:57:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532323744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532323748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732532263740, max timestamp=1732532263749Num uniques 859; 2024-11-25T10:57:43,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:43,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732532263740, max timestamp=1732532263749Num uniques 859; 2024-11-25T10:57:43,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:43,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732532263740, max timestamp=1732532263749Num uniques 859; 2024-11-25T10:57:43,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T10:57:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532323777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532323777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532323782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T10:57:43,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:43,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532323863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:43,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532323863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:43,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532323894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532323894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532323894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:43,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T10:57:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532324073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532324074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532324103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532324103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532324103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,148 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:44,152 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f1f7873d966448f1a566535187e923b0_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f1f7873d966448f1a566535187e923b0_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:44,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9a3d96a748334a7583960bb42cfe5116, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:44,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9a3d96a748334a7583960bb42cfe5116 is 175, key is test_row_0/A:col10/1732532263718/Put/seqid=0 2024-11-25T10:57:44,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742332_1508 (size=30955) 2024-11-25T10:57:44,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:44,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:44,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:44,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:44,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T10:57:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532324384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532324384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532324413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532324413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532324413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9a3d96a748334a7583960bb42cfe5116 2024-11-25T10:57:44,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/f506355ec43c4d769a48715bffea1185 is 50, key is test_row_0/B:col10/1732532263718/Put/seqid=0 2024-11-25T10:57:44,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742333_1509 (size=12001) 2024-11-25T10:57:44,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:44,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:44,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T10:57:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532324894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532324894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532324923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532324923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532324923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:44,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:44,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:44,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/f506355ec43c4d769a48715bffea1185 2024-11-25T10:57:45,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/3537ef0f8ab7480680499293b016e861 is 50, key is test_row_0/C:col10/1732532263718/Put/seqid=0 2024-11-25T10:57:45,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742334_1510 (size=12001) 2024-11-25T10:57:45,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:45,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:45,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:45,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:45,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:45,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:45,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:45,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:45,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:45,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:45,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/3537ef0f8ab7480680499293b016e861 2024-11-25T10:57:45,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9a3d96a748334a7583960bb42cfe5116 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9a3d96a748334a7583960bb42cfe5116 2024-11-25T10:57:45,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9a3d96a748334a7583960bb42cfe5116, entries=150, sequenceid=15, filesize=30.2 K 2024-11-25T10:57:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/f506355ec43c4d769a48715bffea1185 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f506355ec43c4d769a48715bffea1185 2024-11-25T10:57:45,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f506355ec43c4d769a48715bffea1185, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T10:57:45,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/3537ef0f8ab7480680499293b016e861 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/3537ef0f8ab7480680499293b016e861 2024-11-25T10:57:45,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/3537ef0f8ab7480680499293b016e861, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T10:57:45,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fb047475327d71fbf830ed8c3260c99d in 1719ms, sequenceid=15, compaction requested=false 2024-11-25T10:57:45,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:45,549 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T10:57:45,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T10:57:45,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:45,560 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-25T10:57:45,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259be5264a16fd4196801d086811c03fb9_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532263774/Put/seqid=0 2024-11-25T10:57:45,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742335_1511 (size=12154) 2024-11-25T10:57:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T10:57:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:45,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:45,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532325931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:45,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532325934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:45,936 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532325935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532325943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532325943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:45,945 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:45,946 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:45,983 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259be5264a16fd4196801d086811c03fb9_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259be5264a16fd4196801d086811c03fb9_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/85e8084c03b342d9a50387878b9a77e8, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/85e8084c03b342d9a50387878b9a77e8 is 175, key is test_row_0/A:col10/1732532263774/Put/seqid=0 2024-11-25T10:57:45,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742336_1512 (size=30955) 2024-11-25T10:57:45,996 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/85e8084c03b342d9a50387878b9a77e8 2024-11-25T10:57:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/1f3196c3eda64cdfb134e6ff6c9ca019 is 50, key is test_row_0/B:col10/1732532263774/Put/seqid=0 2024-11-25T10:57:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742337_1513 (size=12001) 2024-11-25T10:57:46,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532326043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532326043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532326254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532326254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:46,410 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/1f3196c3eda64cdfb134e6ff6c9ca019 2024-11-25T10:57:46,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/f9d4f202fc0d47be811cd56eadd934f8 is 50, key is test_row_0/C:col10/1732532263774/Put/seqid=0 2024-11-25T10:57:46,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742338_1514 (size=12001) 2024-11-25T10:57:46,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532326564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:46,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532326564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:46,854 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/f9d4f202fc0d47be811cd56eadd934f8 2024-11-25T10:57:46,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/85e8084c03b342d9a50387878b9a77e8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/85e8084c03b342d9a50387878b9a77e8 2024-11-25T10:57:46,861 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/85e8084c03b342d9a50387878b9a77e8, entries=150, sequenceid=50, filesize=30.2 K 2024-11-25T10:57:46,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/1f3196c3eda64cdfb134e6ff6c9ca019 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/1f3196c3eda64cdfb134e6ff6c9ca019 2024-11-25T10:57:46,866 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/1f3196c3eda64cdfb134e6ff6c9ca019, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T10:57:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/f9d4f202fc0d47be811cd56eadd934f8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f9d4f202fc0d47be811cd56eadd934f8 2024-11-25T10:57:46,870 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f9d4f202fc0d47be811cd56eadd934f8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T10:57:46,871 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for fb047475327d71fbf830ed8c3260c99d in 1311ms, sequenceid=50, compaction requested=false 2024-11-25T10:57:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-25T10:57:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-25T10:57:46,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-25T10:57:46,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1560 sec 2024-11-25T10:57:46,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 3.1690 sec 2024-11-25T10:57:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:47,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:57:47,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:47,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:47,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:47,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:47,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:47,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:47,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a70b0e4b127f426bbe415cc900db6f3a_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532265934/Put/seqid=0 2024-11-25T10:57:47,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742339_1515 (size=12154) 2024-11-25T10:57:47,105 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:47,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a70b0e4b127f426bbe415cc900db6f3a_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a70b0e4b127f426bbe415cc900db6f3a_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:47,113 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/071de5ca7e2e4e95a9ba5033aa631e1f, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:47,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/071de5ca7e2e4e95a9ba5033aa631e1f is 175, key is test_row_0/A:col10/1732532265934/Put/seqid=0 2024-11-25T10:57:47,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532327121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532327127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742340_1516 (size=30955) 2024-11-25T10:57:47,182 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/071de5ca7e2e4e95a9ba5033aa631e1f 2024-11-25T10:57:47,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/6c80a3ee340d4dee97ee2d0a1012fc00 is 50, key is test_row_0/B:col10/1732532265934/Put/seqid=0 2024-11-25T10:57:47,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742341_1517 (size=12001) 2024-11-25T10:57:47,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/6c80a3ee340d4dee97ee2d0a1012fc00 2024-11-25T10:57:47,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532327234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532327234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/6d18fa7b678f4bc293cc29b2a6545212 is 50, key is test_row_0/C:col10/1732532265934/Put/seqid=0 2024-11-25T10:57:47,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742342_1518 (size=12001) 2024-11-25T10:57:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532327444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532327444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/6d18fa7b678f4bc293cc29b2a6545212 2024-11-25T10:57:47,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/071de5ca7e2e4e95a9ba5033aa631e1f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/071de5ca7e2e4e95a9ba5033aa631e1f 2024-11-25T10:57:47,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/071de5ca7e2e4e95a9ba5033aa631e1f, entries=150, sequenceid=63, filesize=30.2 K 2024-11-25T10:57:47,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/6c80a3ee340d4dee97ee2d0a1012fc00 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c80a3ee340d4dee97ee2d0a1012fc00 2024-11-25T10:57:47,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c80a3ee340d4dee97ee2d0a1012fc00, entries=150, sequenceid=63, filesize=11.7 K 2024-11-25T10:57:47,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/6d18fa7b678f4bc293cc29b2a6545212 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/6d18fa7b678f4bc293cc29b2a6545212 2024-11-25T10:57:47,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/6d18fa7b678f4bc293cc29b2a6545212, entries=150, sequenceid=63, filesize=11.7 K 2024-11-25T10:57:47,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fb047475327d71fbf830ed8c3260c99d in 602ms, sequenceid=63, compaction requested=true 2024-11-25T10:57:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:47,678 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:47,678 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:47,679 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:47,679 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/A is initiating minor compaction (all files) 2024-11-25T10:57:47,679 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/A in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:47,679 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9a3d96a748334a7583960bb42cfe5116, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/85e8084c03b342d9a50387878b9a77e8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/071de5ca7e2e4e95a9ba5033aa631e1f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=90.7 K 2024-11-25T10:57:47,679 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:47,679 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9a3d96a748334a7583960bb42cfe5116, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/85e8084c03b342d9a50387878b9a77e8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/071de5ca7e2e4e95a9ba5033aa631e1f] 2024-11-25T10:57:47,680 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:47,680 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/B is initiating minor compaction (all files) 2024-11-25T10:57:47,680 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/B in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:47,680 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f506355ec43c4d769a48715bffea1185, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/1f3196c3eda64cdfb134e6ff6c9ca019, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c80a3ee340d4dee97ee2d0a1012fc00] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.2 K 2024-11-25T10:57:47,680 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a3d96a748334a7583960bb42cfe5116, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732532263714 2024-11-25T10:57:47,681 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f506355ec43c4d769a48715bffea1185, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732532263714 2024-11-25T10:57:47,681 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e8084c03b342d9a50387878b9a77e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732532263774 2024-11-25T10:57:47,681 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3196c3eda64cdfb134e6ff6c9ca019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732532263774 2024-11-25T10:57:47,681 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071de5ca7e2e4e95a9ba5033aa631e1f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732532265934 2024-11-25T10:57:47,682 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c80a3ee340d4dee97ee2d0a1012fc00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732532265934 2024-11-25T10:57:47,690 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:47,696 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#B#compaction#442 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:47,697 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/650935cde14d4f32829dc36561df9cb9 is 50, key is test_row_0/B:col10/1732532265934/Put/seqid=0 2024-11-25T10:57:47,701 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125c197f96d3cd6430d980d4eab3d8fa42c_fb047475327d71fbf830ed8c3260c99d store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:47,703 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125c197f96d3cd6430d980d4eab3d8fa42c_fb047475327d71fbf830ed8c3260c99d, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:47,704 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c197f96d3cd6430d980d4eab3d8fa42c_fb047475327d71fbf830ed8c3260c99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:47,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742343_1519 (size=12104) 2024-11-25T10:57:47,752 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/650935cde14d4f32829dc36561df9cb9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/650935cde14d4f32829dc36561df9cb9 2024-11-25T10:57:47,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742344_1520 (size=4469) 2024-11-25T10:57:47,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-25T10:57:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:47,761 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#A#compaction#441 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:47,761 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/2c2183bcaf1649aabc846bbb7acc10bf is 175, key is test_row_0/A:col10/1732532265934/Put/seqid=0 2024-11-25T10:57:47,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532267119, max timestamp=1732532267758Num uniques 859; 2024-11-25T10:57:47,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:47,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:47,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:47,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:47,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:47,764 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/B of fb047475327d71fbf830ed8c3260c99d into 650935cde14d4f32829dc36561df9cb9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:47,764 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:47,764 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/B, priority=13, startTime=1732532267678; duration=0sec 2024-11-25T10:57:47,764 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:47,764 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:B 2024-11-25T10:57:47,764 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:47,767 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:47,767 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/C is initiating minor compaction (all files) 2024-11-25T10:57:47,767 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/C in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:47,767 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/3537ef0f8ab7480680499293b016e861, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f9d4f202fc0d47be811cd56eadd934f8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/6d18fa7b678f4bc293cc29b2a6545212] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.2 K 2024-11-25T10:57:47,768 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3537ef0f8ab7480680499293b016e861, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732532263714 2024-11-25T10:57:47,769 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d4f202fc0d47be811cd56eadd934f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732532263774 2024-11-25T10:57:47,769 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d18fa7b678f4bc293cc29b2a6545212, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732532265934 2024-11-25T10:57:47,800 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#C#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:47,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112529b129ded9e84957ad788c0901064c06_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532267121/Put/seqid=0 2024-11-25T10:57:47,801 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/1c78df60493740d79698081e189d6ece is 50, key is test_row_0/C:col10/1732532265934/Put/seqid=0 2024-11-25T10:57:47,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742345_1521 (size=31058) 2024-11-25T10:57:47,816 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/2c2183bcaf1649aabc846bbb7acc10bf as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2c2183bcaf1649aabc846bbb7acc10bf 2024-11-25T10:57:47,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742347_1523 (size=14594) 2024-11-25T10:57:47,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742346_1522 (size=12104) 2024-11-25T10:57:47,826 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/A of fb047475327d71fbf830ed8c3260c99d into 2c2183bcaf1649aabc846bbb7acc10bf(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:47,826 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:47,826 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/A, priority=13, startTime=1732532267678; duration=0sec 2024-11-25T10:57:47,826 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:47,826 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:A 2024-11-25T10:57:47,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:47,835 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/1c78df60493740d79698081e189d6ece as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1c78df60493740d79698081e189d6ece 2024-11-25T10:57:47,840 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/C of fb047475327d71fbf830ed8c3260c99d into 1c78df60493740d79698081e189d6ece(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:47,840 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:47,840 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/C, priority=13, startTime=1732532267678; duration=0sec 2024-11-25T10:57:47,841 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:47,841 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:C 2024-11-25T10:57:47,845 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112529b129ded9e84957ad788c0901064c06_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112529b129ded9e84957ad788c0901064c06_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:47,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/60ee3a32a5ca4191b6b43962747a026e, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:47,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/60ee3a32a5ca4191b6b43962747a026e is 175, key is test_row_0/A:col10/1732532267121/Put/seqid=0 2024-11-25T10:57:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T10:57:47,878 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:47,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-25T10:57:47,881 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T10:57:47,881 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:47,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:47,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742348_1524 (size=39549) 2024-11-25T10:57:47,897 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/60ee3a32a5ca4191b6b43962747a026e 2024-11-25T10:57:47,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532327898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532327900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/bd89b05faf314dc2a9b67955977d88c0 is 50, key is test_row_0/B:col10/1732532267121/Put/seqid=0 2024-11-25T10:57:47,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742349_1525 (size=12001) 2024-11-25T10:57:47,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/bd89b05faf314dc2a9b67955977d88c0 2024-11-25T10:57:47,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/457b0854ead74ca292d4ed80ec1a952c is 50, key is test_row_0/C:col10/1732532267121/Put/seqid=0 2024-11-25T10:57:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742350_1526 (size=12001) 2024-11-25T10:57:47,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/457b0854ead74ca292d4ed80ec1a952c 2024-11-25T10:57:47,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/60ee3a32a5ca4191b6b43962747a026e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/60ee3a32a5ca4191b6b43962747a026e 2024-11-25T10:57:47,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532327954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,959 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/60ee3a32a5ca4191b6b43962747a026e, entries=200, sequenceid=85, filesize=38.6 K 2024-11-25T10:57:47,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/bd89b05faf314dc2a9b67955977d88c0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/bd89b05faf314dc2a9b67955977d88c0 2024-11-25T10:57:47,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532327963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532327963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:47,968 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,969 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:47,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/bd89b05faf314dc2a9b67955977d88c0, entries=150, sequenceid=85, filesize=11.7 K 2024-11-25T10:57:47,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/457b0854ead74ca292d4ed80ec1a952c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/457b0854ead74ca292d4ed80ec1a952c 2024-11-25T10:57:47,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/457b0854ead74ca292d4ed80ec1a952c, entries=150, sequenceid=85, filesize=11.7 K 2024-11-25T10:57:47,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for fb047475327d71fbf830ed8c3260c99d in 220ms, sequenceid=85, compaction requested=false 2024-11-25T10:57:47,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T10:57:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:48,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:57:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:48,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:48,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bc86a2c198704c3fbe46faaf4238fffb_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532268018/Put/seqid=0 2024-11-25T10:57:48,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T10:57:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742351_1527 (size=14594) 2024-11-25T10:57:48,047 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:48,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bc86a2c198704c3fbe46faaf4238fffb_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bc86a2c198704c3fbe46faaf4238fffb_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:48,057 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/51a08cf6b7b84b249997f56d542fa704, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:48,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/51a08cf6b7b84b249997f56d542fa704 is 175, key is test_row_0/A:col10/1732532268018/Put/seqid=0 2024-11-25T10:57:48,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742352_1528 (size=39549) 2024-11-25T10:57:48,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532328057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532328058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/51a08cf6b7b84b249997f56d542fa704 2024-11-25T10:57:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/2f6f75e9e420487884d2e6e1ca85cbb1 is 50, key is test_row_0/B:col10/1732532268018/Put/seqid=0 2024-11-25T10:57:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742353_1529 (size=12001) 2024-11-25T10:57:48,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532328174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532328174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T10:57:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T10:57:48,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T10:57:48,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:48,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532328383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532328384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/2f6f75e9e420487884d2e6e1ca85cbb1 2024-11-25T10:57:48,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/e22b355e35e94b38b175fded7348c798 is 50, key is test_row_0/C:col10/1732532268018/Put/seqid=0 2024-11-25T10:57:48,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T10:57:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T10:57:48,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742354_1530 (size=12001) 2024-11-25T10:57:48,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T10:57:48,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:48,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532328693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532328693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:48,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T10:57:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:48,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/e22b355e35e94b38b175fded7348c798 2024-11-25T10:57:48,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/51a08cf6b7b84b249997f56d542fa704 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/51a08cf6b7b84b249997f56d542fa704 2024-11-25T10:57:48,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/51a08cf6b7b84b249997f56d542fa704, entries=200, sequenceid=105, filesize=38.6 K 2024-11-25T10:57:48,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/2f6f75e9e420487884d2e6e1ca85cbb1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/2f6f75e9e420487884d2e6e1ca85cbb1 2024-11-25T10:57:48,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/2f6f75e9e420487884d2e6e1ca85cbb1, entries=150, sequenceid=105, filesize=11.7 K 2024-11-25T10:57:48,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/e22b355e35e94b38b175fded7348c798 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/e22b355e35e94b38b175fded7348c798 2024-11-25T10:57:48,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/e22b355e35e94b38b175fded7348c798, entries=150, sequenceid=105, filesize=11.7 K 2024-11-25T10:57:48,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fb047475327d71fbf830ed8c3260c99d in 907ms, sequenceid=105, compaction requested=true 2024-11-25T10:57:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:48,924 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:48,924 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/A is initiating minor compaction (all files) 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/B is initiating minor compaction (all files) 2024-11-25T10:57:48,928 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/A in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,928 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/B in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,928 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2c2183bcaf1649aabc846bbb7acc10bf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/60ee3a32a5ca4191b6b43962747a026e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/51a08cf6b7b84b249997f56d542fa704] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=107.6 K 2024-11-25T10:57:48,928 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/650935cde14d4f32829dc36561df9cb9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/bd89b05faf314dc2a9b67955977d88c0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/2f6f75e9e420487884d2e6e1ca85cbb1] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.3 K 2024-11-25T10:57:48,928 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2c2183bcaf1649aabc846bbb7acc10bf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/60ee3a32a5ca4191b6b43962747a026e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/51a08cf6b7b84b249997f56d542fa704] 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 650935cde14d4f32829dc36561df9cb9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732532265934 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c2183bcaf1649aabc846bbb7acc10bf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732532265934 2024-11-25T10:57:48,928 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting bd89b05faf314dc2a9b67955977d88c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732532267121 2024-11-25T10:57:48,929 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ee3a32a5ca4191b6b43962747a026e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732532267119 2024-11-25T10:57:48,929 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6f75e9e420487884d2e6e1ca85cbb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732532267863 2024-11-25T10:57:48,929 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a08cf6b7b84b249997f56d542fa704, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732532267817 2024-11-25T10:57:48,936 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:48,937 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:48,938 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/176f8eda55864d98a023ed09d871cc40 is 50, key is test_row_0/B:col10/1732532268018/Put/seqid=0 2024-11-25T10:57:48,954 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125030be2f7d9894232b32fb51c01b452d0_fb047475327d71fbf830ed8c3260c99d store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:48,957 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125030be2f7d9894232b32fb51c01b452d0_fb047475327d71fbf830ed8c3260c99d, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:48,957 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125030be2f7d9894232b32fb51c01b452d0_fb047475327d71fbf830ed8c3260c99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:48,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742355_1531 (size=12207) 2024-11-25T10:57:48,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T10:57:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,959 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T10:57:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:48,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:48,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742356_1532 (size=4469) 2024-11-25T10:57:48,964 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#A#compaction#451 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:48,965 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/c077f4e6483d42c296ad4d0b804bfdcb is 175, key is test_row_0/A:col10/1732532268018/Put/seqid=0 2024-11-25T10:57:48,968 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/176f8eda55864d98a023ed09d871cc40 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/176f8eda55864d98a023ed09d871cc40 2024-11-25T10:57:48,973 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/B of fb047475327d71fbf830ed8c3260c99d into 176f8eda55864d98a023ed09d871cc40(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:48,973 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:48,973 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/B, priority=13, startTime=1732532268924; duration=0sec 2024-11-25T10:57:48,973 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:48,973 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:B 2024-11-25T10:57:48,973 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:48,975 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:48,975 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/C is initiating minor compaction (all files) 2024-11-25T10:57:48,975 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/C in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:48,975 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1c78df60493740d79698081e189d6ece, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/457b0854ead74ca292d4ed80ec1a952c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/e22b355e35e94b38b175fded7348c798] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.3 K 2024-11-25T10:57:48,976 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c78df60493740d79698081e189d6ece, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732532265934 2024-11-25T10:57:48,976 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 457b0854ead74ca292d4ed80ec1a952c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732532267121 2024-11-25T10:57:48,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259856dfcabaf04f18bd3878e7e43a9894_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532268055/Put/seqid=0 2024-11-25T10:57:48,977 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e22b355e35e94b38b175fded7348c798, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732532267863 2024-11-25T10:57:48,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742357_1533 (size=31161) 2024-11-25T10:57:48,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742358_1534 (size=12154) 2024-11-25T10:57:48,984 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:48,985 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/4e8567dca2b1441fb5842775027025ec is 50, key is test_row_0/C:col10/1732532268018/Put/seqid=0 2024-11-25T10:57:48,990 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/c077f4e6483d42c296ad4d0b804bfdcb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c077f4e6483d42c296ad4d0b804bfdcb 2024-11-25T10:57:48,994 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/A of fb047475327d71fbf830ed8c3260c99d into c077f4e6483d42c296ad4d0b804bfdcb(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:48,994 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:48,994 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/A, priority=13, startTime=1732532268924; duration=0sec 2024-11-25T10:57:48,994 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:48,994 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:A 2024-11-25T10:57:48,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742359_1535 (size=12207) 2024-11-25T10:57:49,002 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/4e8567dca2b1441fb5842775027025ec as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4e8567dca2b1441fb5842775027025ec 2024-11-25T10:57:49,008 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/C of fb047475327d71fbf830ed8c3260c99d into 4e8567dca2b1441fb5842775027025ec(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:49,008 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:49,008 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/C, priority=13, startTime=1732532268924; duration=0sec 2024-11-25T10:57:49,008 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:49,008 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:C 2024-11-25T10:57:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T10:57:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:49,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:49,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532329238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532329242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:49,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532329354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532329354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:49,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:49,386 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259856dfcabaf04f18bd3878e7e43a9894_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259856dfcabaf04f18bd3878e7e43a9894_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0f788aa87b204ba8b6c6ac8c288ae8d6, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:49,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0f788aa87b204ba8b6c6ac8c288ae8d6 is 175, key is test_row_0/A:col10/1732532268055/Put/seqid=0 2024-11-25T10:57:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742360_1536 (size=30955) 2024-11-25T10:57:49,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532329563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532329563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:49,792 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0f788aa87b204ba8b6c6ac8c288ae8d6 2024-11-25T10:57:49,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b6becf6a2166473ea543281a47819515 is 50, key is test_row_0/B:col10/1732532268055/Put/seqid=0 2024-11-25T10:57:49,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742361_1537 (size=12001) 2024-11-25T10:57:49,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532329873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532329873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T10:57:50,213 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b6becf6a2166473ea543281a47819515 2024-11-25T10:57:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/d31b1361bb7749b6ab022696b7c0c2b1 is 50, key is test_row_0/C:col10/1732532268055/Put/seqid=0 2024-11-25T10:57:50,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742362_1538 (size=12001) 2024-11-25T10:57:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532330383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532330383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:50,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:50,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:50,625 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/d31b1361bb7749b6ab022696b7c0c2b1 2024-11-25T10:57:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0f788aa87b204ba8b6c6ac8c288ae8d6 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0f788aa87b204ba8b6c6ac8c288ae8d6 2024-11-25T10:57:50,633 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0f788aa87b204ba8b6c6ac8c288ae8d6, entries=150, sequenceid=119, filesize=30.2 K 2024-11-25T10:57:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b6becf6a2166473ea543281a47819515 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b6becf6a2166473ea543281a47819515 2024-11-25T10:57:50,638 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b6becf6a2166473ea543281a47819515, entries=150, sequenceid=119, filesize=11.7 K 2024-11-25T10:57:50,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/d31b1361bb7749b6ab022696b7c0c2b1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/d31b1361bb7749b6ab022696b7c0c2b1 2024-11-25T10:57:50,642 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/d31b1361bb7749b6ab022696b7c0c2b1, entries=150, sequenceid=119, filesize=11.7 K 2024-11-25T10:57:50,643 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fb047475327d71fbf830ed8c3260c99d in 1684ms, sequenceid=119, compaction requested=false 2024-11-25T10:57:50,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:50,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:50,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-25T10:57:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-25T10:57:50,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-25T10:57:50,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7630 sec 2024-11-25T10:57:50,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.7680 sec 2024-11-25T10:57:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:51,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T10:57:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:51,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250fe39edb834145f788b704de4f3be208_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532269228/Put/seqid=0 2024-11-25T10:57:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742363_1539 (size=14794) 2024-11-25T10:57:51,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532331435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532331435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532331543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532331543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532331753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532331753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:51,823 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:51,828 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250fe39edb834145f788b704de4f3be208_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250fe39edb834145f788b704de4f3be208_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:51,829 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/585bd6dfa84d4cfeb709d11f21bc2c9f, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:51,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/585bd6dfa84d4cfeb709d11f21bc2c9f is 175, key is test_row_0/A:col10/1732532269228/Put/seqid=0 2024-11-25T10:57:51,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742364_1540 (size=39749) 2024-11-25T10:57:51,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/585bd6dfa84d4cfeb709d11f21bc2c9f 2024-11-25T10:57:51,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/98ae5dc527ff480a99d932aa02eb51cb is 50, key is test_row_0/B:col10/1732532269228/Put/seqid=0 2024-11-25T10:57:51,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742365_1541 (size=12151) 2024-11-25T10:57:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532331973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,975 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:51,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532331983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:51,987 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:52,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532332014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:52,016 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:57:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T10:57:52,033 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:52,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-25T10:57:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T10:57:52,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:52,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:52,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532332063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532332063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T10:57:52,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T10:57:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:52,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/98ae5dc527ff480a99d932aa02eb51cb 2024-11-25T10:57:52,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a10bb8d525f74be4b6cbea04c7a35242 is 50, key is test_row_0/C:col10/1732532269228/Put/seqid=0 2024-11-25T10:57:52,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742366_1542 (size=12151) 2024-11-25T10:57:52,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T10:57:52,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:52,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T10:57:52,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T10:57:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532332574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:52,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532332574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:52,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T10:57:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T10:57:52,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a10bb8d525f74be4b6cbea04c7a35242 2024-11-25T10:57:52,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/585bd6dfa84d4cfeb709d11f21bc2c9f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/585bd6dfa84d4cfeb709d11f21bc2c9f 2024-11-25T10:57:52,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/585bd6dfa84d4cfeb709d11f21bc2c9f, entries=200, sequenceid=142, filesize=38.8 K 2024-11-25T10:57:52,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/98ae5dc527ff480a99d932aa02eb51cb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/98ae5dc527ff480a99d932aa02eb51cb 2024-11-25T10:57:52,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/98ae5dc527ff480a99d932aa02eb51cb, entries=150, sequenceid=142, filesize=11.9 K 2024-11-25T10:57:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a10bb8d525f74be4b6cbea04c7a35242 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a10bb8d525f74be4b6cbea04c7a35242 2024-11-25T10:57:52,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a10bb8d525f74be4b6cbea04c7a35242, entries=150, sequenceid=142, filesize=11.9 K 2024-11-25T10:57:52,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for fb047475327d71fbf830ed8c3260c99d in 1303ms, sequenceid=142, compaction requested=true 2024-11-25T10:57:52,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:52,710 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:52,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:52,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:52,710 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:52,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:52,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:52,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:52,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:52,712 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:52,712 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/A is initiating minor compaction (all files) 2024-11-25T10:57:52,712 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/A in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,712 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c077f4e6483d42c296ad4d0b804bfdcb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0f788aa87b204ba8b6c6ac8c288ae8d6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/585bd6dfa84d4cfeb709d11f21bc2c9f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=99.5 K 2024-11-25T10:57:52,712 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,712 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c077f4e6483d42c296ad4d0b804bfdcb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0f788aa87b204ba8b6c6ac8c288ae8d6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/585bd6dfa84d4cfeb709d11f21bc2c9f] 2024-11-25T10:57:52,712 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c077f4e6483d42c296ad4d0b804bfdcb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732532267863 2024-11-25T10:57:52,712 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:52,712 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/B is initiating minor compaction (all files) 2024-11-25T10:57:52,713 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/B in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,713 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/176f8eda55864d98a023ed09d871cc40, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b6becf6a2166473ea543281a47819515, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/98ae5dc527ff480a99d932aa02eb51cb] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.5 K 2024-11-25T10:57:52,713 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f788aa87b204ba8b6c6ac8c288ae8d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732532268048 2024-11-25T10:57:52,713 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585bd6dfa84d4cfeb709d11f21bc2c9f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532269228 2024-11-25T10:57:52,713 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 176f8eda55864d98a023ed09d871cc40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732532267863 2024-11-25T10:57:52,714 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting b6becf6a2166473ea543281a47819515, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732532268048 2024-11-25T10:57:52,714 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ae5dc527ff480a99d932aa02eb51cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532269228 2024-11-25T10:57:52,718 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:52,721 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125f1502f640c7341a48c8eee453c6eddb3_fb047475327d71fbf830ed8c3260c99d store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:52,723 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125f1502f640c7341a48c8eee453c6eddb3_fb047475327d71fbf830ed8c3260c99d, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:52,723 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f1502f640c7341a48c8eee453c6eddb3_fb047475327d71fbf830ed8c3260c99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:52,727 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#B#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:52,727 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/8c463cb8baf64aebbd18aac20fc92bf9 is 50, key is test_row_0/B:col10/1732532269228/Put/seqid=0 2024-11-25T10:57:52,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742368_1544 (size=12459) 2024-11-25T10:57:52,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742367_1543 (size=4469) 2024-11-25T10:57:52,757 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/8c463cb8baf64aebbd18aac20fc92bf9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/8c463cb8baf64aebbd18aac20fc92bf9 2024-11-25T10:57:52,762 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/B of fb047475327d71fbf830ed8c3260c99d into 8c463cb8baf64aebbd18aac20fc92bf9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:52,762 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:52,763 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/B, priority=13, startTime=1732532272710; duration=0sec 2024-11-25T10:57:52,763 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:52,763 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:B 2024-11-25T10:57:52,763 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:52,764 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:52,764 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/C is initiating minor compaction (all files) 2024-11-25T10:57:52,764 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/C in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,764 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4e8567dca2b1441fb5842775027025ec, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/d31b1361bb7749b6ab022696b7c0c2b1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a10bb8d525f74be4b6cbea04c7a35242] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.5 K 2024-11-25T10:57:52,765 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e8567dca2b1441fb5842775027025ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732532267863 2024-11-25T10:57:52,765 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d31b1361bb7749b6ab022696b7c0c2b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732532268048 2024-11-25T10:57:52,766 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a10bb8d525f74be4b6cbea04c7a35242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532269228 2024-11-25T10:57:52,773 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:52,775 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/77d562cf4ada4ef08fe097063d61de56 is 50, key is test_row_0/C:col10/1732532269228/Put/seqid=0 2024-11-25T10:57:52,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742369_1545 (size=12459) 2024-11-25T10:57:52,785 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/77d562cf4ada4ef08fe097063d61de56 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/77d562cf4ada4ef08fe097063d61de56 2024-11-25T10:57:52,791 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/C of fb047475327d71fbf830ed8c3260c99d into 77d562cf4ada4ef08fe097063d61de56(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:52,791 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:52,791 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/C, priority=13, startTime=1732532272711; duration=0sec 2024-11-25T10:57:52,791 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:52,791 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:C 2024-11-25T10:57:52,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T10:57:52,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:52,802 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T10:57:52,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:52,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:52,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:52,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:52,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:52,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252bc4c70060ec4464ba12e6abcd1a7781_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532271432/Put/seqid=0 2024-11-25T10:57:52,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742370_1546 (size=12304) 2024-11-25T10:57:53,139 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#A#compaction#459 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:53,140 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/af893bccba564b87885a52012525ef71 is 175, key is test_row_0/A:col10/1732532269228/Put/seqid=0 2024-11-25T10:57:53,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742371_1547 (size=31413) 2024-11-25T10:57:53,160 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/af893bccba564b87885a52012525ef71 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/af893bccba564b87885a52012525ef71 2024-11-25T10:57:53,165 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/A of fb047475327d71fbf830ed8c3260c99d into af893bccba564b87885a52012525ef71(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:53,165 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:53,165 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/A, priority=13, startTime=1732532272710; duration=0sec 2024-11-25T10:57:53,165 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:53,165 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:A 2024-11-25T10:57:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T10:57:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:53,239 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252bc4c70060ec4464ba12e6abcd1a7781_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252bc4c70060ec4464ba12e6abcd1a7781_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:53,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9b9119034a374b4e97ae58111d32681e, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9b9119034a374b4e97ae58111d32681e is 175, key is test_row_0/A:col10/1732532271432/Put/seqid=0 2024-11-25T10:57:53,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742372_1548 (size=31105) 2024-11-25T10:57:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:53,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532333625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:53,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532333629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:53,646 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9b9119034a374b4e97ae58111d32681e 2024-11-25T10:57:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/56ce5d837c7f49ba9ea504b85e8bea52 is 50, key is test_row_0/B:col10/1732532271432/Put/seqid=0 2024-11-25T10:57:53,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742373_1549 (size=12151) 2024-11-25T10:57:53,658 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/56ce5d837c7f49ba9ea504b85e8bea52 2024-11-25T10:57:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/5f9e7978e0644af6a5083d842b006d89 is 50, key is test_row_0/C:col10/1732532271432/Put/seqid=0 2024-11-25T10:57:53,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742374_1550 (size=12151) 2024-11-25T10:57:53,668 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/5f9e7978e0644af6a5083d842b006d89 2024-11-25T10:57:53,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/9b9119034a374b4e97ae58111d32681e as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9b9119034a374b4e97ae58111d32681e 2024-11-25T10:57:53,677 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9b9119034a374b4e97ae58111d32681e, entries=150, sequenceid=157, filesize=30.4 K 2024-11-25T10:57:53,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/56ce5d837c7f49ba9ea504b85e8bea52 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/56ce5d837c7f49ba9ea504b85e8bea52 2024-11-25T10:57:53,682 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/56ce5d837c7f49ba9ea504b85e8bea52, entries=150, sequenceid=157, filesize=11.9 K 2024-11-25T10:57:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/5f9e7978e0644af6a5083d842b006d89 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/5f9e7978e0644af6a5083d842b006d89 2024-11-25T10:57:53,686 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/5f9e7978e0644af6a5083d842b006d89, entries=150, sequenceid=157, filesize=11.9 K 2024-11-25T10:57:53,687 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fb047475327d71fbf830ed8c3260c99d in 885ms, sequenceid=157, compaction requested=false 2024-11-25T10:57:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-25T10:57:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-25T10:57:53,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-25T10:57:53,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6520 sec 2024-11-25T10:57:53,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.6570 sec 2024-11-25T10:57:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:53,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T10:57:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:53,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c50431e45b574d3989a71dfc1e7cd925_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532273624/Put/seqid=0 2024-11-25T10:57:53,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742375_1551 (size=14794) 2024-11-25T10:57:53,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532333761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:53,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532333762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:53,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532333873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532333873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:54,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532334083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532334083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:54,147 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:54,151 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c50431e45b574d3989a71dfc1e7cd925_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c50431e45b574d3989a71dfc1e7cd925_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:54,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b96b4070b770456abcc24054eb37c3a3, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:54,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b96b4070b770456abcc24054eb37c3a3 is 175, key is test_row_0/A:col10/1732532273624/Put/seqid=0 2024-11-25T10:57:54,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742376_1552 (size=39749) 2024-11-25T10:57:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T10:57:54,193 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:54,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-25T10:57:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T10:57:54,195 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:54,196 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:54,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T10:57:54,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532334393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:54,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532334393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:54,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T10:57:54,560 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b96b4070b770456abcc24054eb37c3a3 2024-11-25T10:57:54,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/6c5310a5d5014a43a01994f17197c07c is 50, key is test_row_0/B:col10/1732532273624/Put/seqid=0 2024-11-25T10:57:54,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742377_1553 (size=12151) 2024-11-25T10:57:54,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:54,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T10:57:54,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532334903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:54,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532334903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:54,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:54,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:54,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/6c5310a5d5014a43a01994f17197c07c 2024-11-25T10:57:54,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/355ea117afcb44f6ac8849ec4523b716 is 50, key is test_row_0/C:col10/1732532273624/Put/seqid=0 2024-11-25T10:57:54,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742378_1554 (size=12151) 2024-11-25T10:57:55,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:55,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T10:57:55,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/355ea117afcb44f6ac8849ec4523b716 2024-11-25T10:57:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b96b4070b770456abcc24054eb37c3a3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b96b4070b770456abcc24054eb37c3a3 2024-11-25T10:57:55,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b96b4070b770456abcc24054eb37c3a3, entries=200, sequenceid=178, filesize=38.8 K 2024-11-25T10:57:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/6c5310a5d5014a43a01994f17197c07c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c5310a5d5014a43a01994f17197c07c 2024-11-25T10:57:55,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c5310a5d5014a43a01994f17197c07c, entries=150, sequenceid=178, filesize=11.9 K 2024-11-25T10:57:55,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/355ea117afcb44f6ac8849ec4523b716 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/355ea117afcb44f6ac8849ec4523b716 2024-11-25T10:57:55,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/355ea117afcb44f6ac8849ec4523b716, entries=150, sequenceid=178, filesize=11.9 K 2024-11-25T10:57:55,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fb047475327d71fbf830ed8c3260c99d in 1672ms, sequenceid=178, compaction requested=true 2024-11-25T10:57:55,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:55,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:55,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:55,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:55,408 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:55,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:55,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:55,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:57:55,408 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:55,408 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:55,409 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/A is initiating minor compaction (all files) 2024-11-25T10:57:55,409 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/A in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,409 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/af893bccba564b87885a52012525ef71, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9b9119034a374b4e97ae58111d32681e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b96b4070b770456abcc24054eb37c3a3] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=99.9 K 2024-11-25T10:57:55,409 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,409 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/af893bccba564b87885a52012525ef71, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9b9119034a374b4e97ae58111d32681e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b96b4070b770456abcc24054eb37c3a3] 2024-11-25T10:57:55,411 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:55,411 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/B is initiating minor compaction (all files) 2024-11-25T10:57:55,411 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/B in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,411 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/8c463cb8baf64aebbd18aac20fc92bf9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/56ce5d837c7f49ba9ea504b85e8bea52, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c5310a5d5014a43a01994f17197c07c] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.9 K 2024-11-25T10:57:55,411 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting af893bccba564b87885a52012525ef71, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532269228 2024-11-25T10:57:55,412 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c463cb8baf64aebbd18aac20fc92bf9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532269228 2024-11-25T10:57:55,412 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9119034a374b4e97ae58111d32681e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532271423 2024-11-25T10:57:55,412 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ce5d837c7f49ba9ea504b85e8bea52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532271423 2024-11-25T10:57:55,412 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b96b4070b770456abcc24054eb37c3a3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732532273618 2024-11-25T10:57:55,413 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5310a5d5014a43a01994f17197c07c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732532273618 2024-11-25T10:57:55,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T10:57:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,421 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T10:57:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:55,423 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:55,423 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/e6e5b085710644848ef99d542baaa55c is 50, key is test_row_0/B:col10/1732532273624/Put/seqid=0 2024-11-25T10:57:55,429 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:55,435 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125e235be5c18ab4bf6a166df7cb41dfe70_fb047475327d71fbf830ed8c3260c99d store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:55,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251459eb2a215747d495f1d9fd1a97ed00_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532273761/Put/seqid=0 2024-11-25T10:57:55,438 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125e235be5c18ab4bf6a166df7cb41dfe70_fb047475327d71fbf830ed8c3260c99d, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:55,438 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e235be5c18ab4bf6a166df7cb41dfe70_fb047475327d71fbf830ed8c3260c99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:55,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742379_1555 (size=12561) 2024-11-25T10:57:55,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742380_1556 (size=12304) 2024-11-25T10:57:55,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742381_1557 (size=4469) 2024-11-25T10:57:55,854 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/e6e5b085710644848ef99d542baaa55c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/e6e5b085710644848ef99d542baaa55c 2024-11-25T10:57:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,859 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251459eb2a215747d495f1d9fd1a97ed00_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251459eb2a215747d495f1d9fd1a97ed00_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:55,859 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/B of fb047475327d71fbf830ed8c3260c99d into e6e5b085710644848ef99d542baaa55c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:55,859 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:55,859 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:55,859 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/B, priority=13, startTime=1732532275407; duration=0sec 2024-11-25T10:57:55,859 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:55,859 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:B 2024-11-25T10:57:55,860 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:55,860 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/c153105448d94c3c88093c94096613a3 is 175, key is test_row_0/A:col10/1732532273624/Put/seqid=0 2024-11-25T10:57:55,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0c4a2f4a198445d9afd414e29a8b2603, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:55,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0c4a2f4a198445d9afd414e29a8b2603 is 175, key is test_row_0/A:col10/1732532273761/Put/seqid=0 2024-11-25T10:57:55,863 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:55,863 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/C is initiating minor compaction (all files) 2024-11-25T10:57:55,863 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/C in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,863 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/77d562cf4ada4ef08fe097063d61de56, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/5f9e7978e0644af6a5083d842b006d89, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/355ea117afcb44f6ac8849ec4523b716] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=35.9 K 2024-11-25T10:57:55,864 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d562cf4ada4ef08fe097063d61de56, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732532269228 2024-11-25T10:57:55,864 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f9e7978e0644af6a5083d842b006d89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732532271423 2024-11-25T10:57:55,865 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 355ea117afcb44f6ac8849ec4523b716, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732532273618 2024-11-25T10:57:55,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742382_1558 (size=31515) 2024-11-25T10:57:55,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742383_1559 (size=31105) 2024-11-25T10:57:55,871 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0c4a2f4a198445d9afd414e29a8b2603 2024-11-25T10:57:55,872 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/c153105448d94c3c88093c94096613a3 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c153105448d94c3c88093c94096613a3 2024-11-25T10:57:55,876 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:55,877 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/506a06319ece45759247ff29d814b008 is 50, key is test_row_0/C:col10/1732532273624/Put/seqid=0 2024-11-25T10:57:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/d8f1d8aeaa614087899e04ce6d860da5 is 50, key is test_row_0/B:col10/1732532273761/Put/seqid=0 2024-11-25T10:57:55,880 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/A of fb047475327d71fbf830ed8c3260c99d into c153105448d94c3c88093c94096613a3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:55,880 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:55,880 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/A, priority=13, startTime=1732532275407; duration=0sec 2024-11-25T10:57:55,880 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:55,880 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:A 2024-11-25T10:57:55,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742385_1561 (size=12151) 2024-11-25T10:57:55,886 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/d8f1d8aeaa614087899e04ce6d860da5 2024-11-25T10:57:55,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742384_1560 (size=12561) 2024-11-25T10:57:55,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/c825c936d9fc41e2b99635df5be9aae7 is 50, key is test_row_0/C:col10/1732532273761/Put/seqid=0 2024-11-25T10:57:55,898 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/506a06319ece45759247ff29d814b008 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/506a06319ece45759247ff29d814b008 2024-11-25T10:57:55,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742386_1562 (size=12151) 2024-11-25T10:57:55,901 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/c825c936d9fc41e2b99635df5be9aae7 2024-11-25T10:57:55,903 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/C of fb047475327d71fbf830ed8c3260c99d into 506a06319ece45759247ff29d814b008(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:55,903 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:55,903 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/C, priority=13, startTime=1732532275408; duration=0sec 2024-11-25T10:57:55,904 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:55,904 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:C 2024-11-25T10:57:55,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/0c4a2f4a198445d9afd414e29a8b2603 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0c4a2f4a198445d9afd414e29a8b2603 2024-11-25T10:57:55,911 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0c4a2f4a198445d9afd414e29a8b2603, entries=150, sequenceid=190, filesize=30.4 K 2024-11-25T10:57:55,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/d8f1d8aeaa614087899e04ce6d860da5 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d8f1d8aeaa614087899e04ce6d860da5 2024-11-25T10:57:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,915 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d8f1d8aeaa614087899e04ce6d860da5, entries=150, sequenceid=190, filesize=11.9 K 2024-11-25T10:57:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/c825c936d9fc41e2b99635df5be9aae7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/c825c936d9fc41e2b99635df5be9aae7 2024-11-25T10:57:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,920 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/c825c936d9fc41e2b99635df5be9aae7, entries=150, sequenceid=190, filesize=11.9 K 2024-11-25T10:57:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,921 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for fb047475327d71fbf830ed8c3260c99d in 501ms, sequenceid=190, compaction requested=false 2024-11-25T10:57:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-25T10:57:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-25T10:57:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-25T10:57:55,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7270 sec 2024-11-25T10:57:55,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.7310 sec 2024-11-25T10:57:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:55,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T10:57:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bf56f1b564b14f17a5bd3cbf4f22bb4b_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532275963/Put/seqid=0 2024-11-25T10:57:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:56,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742387_1563 (size=19774) 2024-11-25T10:57:56,015 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:56,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bf56f1b564b14f17a5bd3cbf4f22bb4b_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bf56f1b564b14f17a5bd3cbf4f22bb4b_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:56,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b51c144bc3b940e08dc8ff271936cf60, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:56,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b51c144bc3b940e08dc8ff271936cf60 is 175, key is test_row_0/A:col10/1732532275963/Put/seqid=0 2024-11-25T10:57:56,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742388_1564 (size=57033) 2024-11-25T10:57:56,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b51c144bc3b940e08dc8ff271936cf60 2024-11-25T10:57:56,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/049994888a7e4b348b47355545485c9d is 50, key is test_row_0/B:col10/1732532275963/Put/seqid=0 2024-11-25T10:57:56,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742389_1565 (size=12151) 2024-11-25T10:57:56,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/049994888a7e4b348b47355545485c9d 2024-11-25T10:57:56,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/332d4f7cd5ef4a3b9af49216fc6b50d8 is 50, key is test_row_0/C:col10/1732532275963/Put/seqid=0 2024-11-25T10:57:56,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532276023, max timestamp=1732532276047Num uniques 859; 2024-11-25T10:57:56,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532276023, max timestamp=1732532276047Num uniques 859; 2024-11-25T10:57:56,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:56,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:56,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732532276023, max timestamp=1732532276047Num uniques 859; 2024-11-25T10:57:56,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6bfc37bad115:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T10:57:56,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742390_1566 (size=12151) 2024-11-25T10:57:56,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/332d4f7cd5ef4a3b9af49216fc6b50d8 2024-11-25T10:57:56,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b51c144bc3b940e08dc8ff271936cf60 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b51c144bc3b940e08dc8ff271936cf60 2024-11-25T10:57:56,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b51c144bc3b940e08dc8ff271936cf60, entries=300, sequenceid=203, filesize=55.7 K 2024-11-25T10:57:56,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/049994888a7e4b348b47355545485c9d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/049994888a7e4b348b47355545485c9d 2024-11-25T10:57:56,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/049994888a7e4b348b47355545485c9d, entries=150, sequenceid=203, filesize=11.9 K 2024-11-25T10:57:56,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/332d4f7cd5ef4a3b9af49216fc6b50d8 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/332d4f7cd5ef4a3b9af49216fc6b50d8 2024-11-25T10:57:56,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/332d4f7cd5ef4a3b9af49216fc6b50d8, entries=150, sequenceid=203, filesize=11.9 K 2024-11-25T10:57:56,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=80.51 KB/82440 for fb047475327d71fbf830ed8c3260c99d in 100ms, sequenceid=203, compaction requested=true 2024-11-25T10:57:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:56,082 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:57:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:56,082 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:57:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:57:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:56,082 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:56,082 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/A is initiating minor compaction (all files) 2024-11-25T10:57:56,083 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/A in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,083 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c153105448d94c3c88093c94096613a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0c4a2f4a198445d9afd414e29a8b2603, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b51c144bc3b940e08dc8ff271936cf60] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=116.8 K 2024-11-25T10:57:56,083 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,083 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c153105448d94c3c88093c94096613a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0c4a2f4a198445d9afd414e29a8b2603, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b51c144bc3b940e08dc8ff271936cf60] 2024-11-25T10:57:56,083 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:56,083 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/B is initiating minor compaction (all files) 2024-11-25T10:57:56,083 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/B in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,083 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/e6e5b085710644848ef99d542baaa55c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d8f1d8aeaa614087899e04ce6d860da5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/049994888a7e4b348b47355545485c9d] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=36.0 K 2024-11-25T10:57:56,084 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c153105448d94c3c88093c94096613a3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732532273618 2024-11-25T10:57:56,084 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e5b085710644848ef99d542baaa55c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732532273618 2024-11-25T10:57:56,084 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c4a2f4a198445d9afd414e29a8b2603, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732532273751 2024-11-25T10:57:56,085 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting d8f1d8aeaa614087899e04ce6d860da5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732532273751 2024-11-25T10:57:56,085 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51c144bc3b940e08dc8ff271936cf60, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732532275953 2024-11-25T10:57:56,085 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 049994888a7e4b348b47355545485c9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732532275963 2024-11-25T10:57:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:56,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=93.93 KB heapSize=265.45 KB 2024-11-25T10:57:56,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:56,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:56,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:56,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:56,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:56,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T10:57:56,092 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:56,092 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b85398a6dd4142e8b60793577192d8ca is 50, key is test_row_0/B:col10/1732532275963/Put/seqid=0 2024-11-25T10:57:56,097 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:56,109 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251732c028b00041abb307d1d6246da5b8_fb047475327d71fbf830ed8c3260c99d store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:56,112 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251732c028b00041abb307d1d6246da5b8_fb047475327d71fbf830ed8c3260c99d, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:56,112 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251732c028b00041abb307d1d6246da5b8_fb047475327d71fbf830ed8c3260c99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:56,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250191517ed8dc4c59a7ccc1e8a38dd5fe_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532276078/Put/seqid=0 2024-11-25T10:57:56,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742391_1567 (size=12663) 2024-11-25T10:57:56,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742392_1568 (size=4469) 2024-11-25T10:57:56,143 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#A#compaction#478 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:56,144 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/859038cb49424afeb3c0132b3a661430 is 175, key is test_row_0/A:col10/1732532275963/Put/seqid=0 2024-11-25T10:57:56,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532336140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532336145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742393_1569 (size=17284) 2024-11-25T10:57:56,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742394_1570 (size=31617) 2024-11-25T10:57:56,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532336253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532336254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T10:57:56,344 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:57:56,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:57:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-25T10:57:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T10:57:56,346 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:57:56,347 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:57:56,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:57:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T10:57:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532336464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532336464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,542 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b85398a6dd4142e8b60793577192d8ca as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b85398a6dd4142e8b60793577192d8ca 2024-11-25T10:57:56,547 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/B of fb047475327d71fbf830ed8c3260c99d into b85398a6dd4142e8b60793577192d8ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:56,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:56,547 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/B, priority=13, startTime=1732532276082; duration=0sec 2024-11-25T10:57:56,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:57:56,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:B 2024-11-25T10:57:56,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T10:57:56,548 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T10:57:56,548 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/C is initiating minor compaction (all files) 2024-11-25T10:57:56,548 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/C in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,548 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/506a06319ece45759247ff29d814b008, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/c825c936d9fc41e2b99635df5be9aae7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/332d4f7cd5ef4a3b9af49216fc6b50d8] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=36.0 K 2024-11-25T10:57:56,548 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 506a06319ece45759247ff29d814b008, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732532273618 2024-11-25T10:57:56,549 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c825c936d9fc41e2b99635df5be9aae7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732532273751 2024-11-25T10:57:56,549 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 332d4f7cd5ef4a3b9af49216fc6b50d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732532275963 2024-11-25T10:57:56,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:56,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250191517ed8dc4c59a7ccc1e8a38dd5fe_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250191517ed8dc4c59a7ccc1e8a38dd5fe_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:56,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/070b9075308c40d59adedb86fff176d9, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:56,555 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:57:56,556 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/1ea38e3768ba48e1a760130d4363e69b is 50, key is test_row_0/C:col10/1732532275963/Put/seqid=0 2024-11-25T10:57:56,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/070b9075308c40d59adedb86fff176d9 is 175, key is test_row_0/A:col10/1732532276078/Put/seqid=0 2024-11-25T10:57:56,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742396_1572 (size=48389) 2024-11-25T10:57:56,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742395_1571 (size=12663) 2024-11-25T10:57:56,568 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/859038cb49424afeb3c0132b3a661430 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/859038cb49424afeb3c0132b3a661430 2024-11-25T10:57:56,573 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/A of fb047475327d71fbf830ed8c3260c99d into 859038cb49424afeb3c0132b3a661430(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:56,573 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:56,573 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/A, priority=13, startTime=1732532276082; duration=0sec 2024-11-25T10:57:56,573 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:56,573 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:A 2024-11-25T10:57:56,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T10:57:56,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532336775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532336775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:56,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:56,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:56,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/070b9075308c40d59adedb86fff176d9 2024-11-25T10:57:56,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:56,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:56,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:56,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T10:57:57,004 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/1ea38e3768ba48e1a760130d4363e69b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1ea38e3768ba48e1a760130d4363e69b 2024-11-25T10:57:57,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/26149a4921bc48b5aca48f1322cd07f7 is 50, key is test_row_0/B:col10/1732532276078/Put/seqid=0 2024-11-25T10:57:57,019 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/C of fb047475327d71fbf830ed8c3260c99d into 1ea38e3768ba48e1a760130d4363e69b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:57:57,019 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:57,019 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/C, priority=13, startTime=1732532276082; duration=0sec 2024-11-25T10:57:57,019 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:57:57,019 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:C 2024-11-25T10:57:57,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742397_1573 (size=12151) 2024-11-25T10:57:57,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532337284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532337294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:57,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/26149a4921bc48b5aca48f1322cd07f7 2024-11-25T10:57:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T10:57:57,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/0b28cc75afa045dcb86ff9f8158e5a11 is 50, key is test_row_0/C:col10/1732532276078/Put/seqid=0 2024-11-25T10:57:57,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742398_1574 (size=12151) 2024-11-25T10:57:57,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:57,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:57,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:57,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:57:57,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/0b28cc75afa045dcb86ff9f8158e5a11 2024-11-25T10:57:58,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/070b9075308c40d59adedb86fff176d9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/070b9075308c40d59adedb86fff176d9 2024-11-25T10:57:58,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/070b9075308c40d59adedb86fff176d9, entries=250, sequenceid=234, filesize=47.3 K 2024-11-25T10:57:58,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/26149a4921bc48b5aca48f1322cd07f7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/26149a4921bc48b5aca48f1322cd07f7 2024-11-25T10:57:58,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/26149a4921bc48b5aca48f1322cd07f7, entries=150, sequenceid=234, filesize=11.9 K 2024-11-25T10:57:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/0b28cc75afa045dcb86ff9f8158e5a11 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/0b28cc75afa045dcb86ff9f8158e5a11 2024-11-25T10:57:58,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/0b28cc75afa045dcb86ff9f8158e5a11, entries=150, sequenceid=234, filesize=11.9 K 2024-11-25T10:57:58,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~265.41 KB/271776, currentSize=87.22 KB/89310 for fb047475327d71fbf830ed8c3260c99d in 1999ms, sequenceid=234, compaction requested=false 2024-11-25T10:57:58,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:58,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T10:57:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:58,093 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T10:57:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:58,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259fe4d13d45e345ef888efec30c38b4ad_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532276142/Put/seqid=0 2024-11-25T10:57:58,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742399_1575 (size=12304) 2024-11-25T10:57:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:58,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:57:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532338364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:58,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532338367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:58,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532338484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:58,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532338493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T10:57:58,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:57:58,596 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259fe4d13d45e345ef888efec30c38b4ad_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259fe4d13d45e345ef888efec30c38b4ad_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:58,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b1c93139d30a4cbda41f4b75b46f3b6f, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:57:58,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b1c93139d30a4cbda41f4b75b46f3b6f is 175, key is test_row_0/A:col10/1732532276142/Put/seqid=0 2024-11-25T10:57:58,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742400_1576 (size=31105) 2024-11-25T10:57:58,641 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b1c93139d30a4cbda41f4b75b46f3b6f 2024-11-25T10:57:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/d0ebc15fe725469793ccdd87dcbe543d is 50, key is test_row_0/B:col10/1732532276142/Put/seqid=0 2024-11-25T10:57:58,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742401_1577 (size=12151) 2024-11-25T10:57:58,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532338705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:58,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532338708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:58,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532339029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532339033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,109 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/d0ebc15fe725469793ccdd87dcbe543d 2024-11-25T10:57:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/4de33e495feb4a9da0fc24a292823629 is 50, key is test_row_0/C:col10/1732532276142/Put/seqid=0 2024-11-25T10:57:59,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742402_1578 (size=12151) 2024-11-25T10:57:59,154 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/4de33e495feb4a9da0fc24a292823629 2024-11-25T10:57:59,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/b1c93139d30a4cbda41f4b75b46f3b6f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b1c93139d30a4cbda41f4b75b46f3b6f 2024-11-25T10:57:59,173 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b1c93139d30a4cbda41f4b75b46f3b6f, entries=150, sequenceid=253, filesize=30.4 K 2024-11-25T10:57:59,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/d0ebc15fe725469793ccdd87dcbe543d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d0ebc15fe725469793ccdd87dcbe543d 2024-11-25T10:57:59,183 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d0ebc15fe725469793ccdd87dcbe543d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T10:57:59,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/4de33e495feb4a9da0fc24a292823629 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4de33e495feb4a9da0fc24a292823629 2024-11-25T10:57:59,215 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4de33e495feb4a9da0fc24a292823629, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T10:57:59,216 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fb047475327d71fbf830ed8c3260c99d in 1123ms, sequenceid=253, compaction requested=true 2024-11-25T10:57:59,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:57:59,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:57:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-25T10:57:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-25T10:57:59,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-25T10:57:59,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8740 sec 2024-11-25T10:57:59,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.8790 sec 2024-11-25T10:57:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:57:59,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T10:57:59,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:57:59,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:59,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:57:59,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:59,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:57:59,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:57:59,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b55c3e7c8eed4431b5b7b7ab93949081_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532278350/Put/seqid=0 2024-11-25T10:57:59,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742403_1579 (size=12454) 2024-11-25T10:57:59,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532339625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532339627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532339745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532339745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532339954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:57:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532339954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:57:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:57:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:57:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:00,036 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b55c3e7c8eed4431b5b7b7ab93949081_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b55c3e7c8eed4431b5b7b7ab93949081_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:00,086 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/32acdde9d5c44bc3a6dd687a553743dd, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:00,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/32acdde9d5c44bc3a6dd687a553743dd is 175, key is test_row_0/A:col10/1732532278350/Put/seqid=0 2024-11-25T10:58:00,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742404_1580 (size=31255) 2024-11-25T10:58:00,133 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/32acdde9d5c44bc3a6dd687a553743dd 2024-11-25T10:58:00,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b9734b5387074155b93266f64ac4dda7 is 50, key is test_row_0/B:col10/1732532278350/Put/seqid=0 2024-11-25T10:58:00,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742405_1581 (size=12301) 2024-11-25T10:58:00,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532340264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:00,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532340264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T10:58:00,551 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:58:00,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T10:58:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-25T10:58:00,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T10:58:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T10:58:00,555 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T10:58:00,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T10:58:00,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b9734b5387074155b93266f64ac4dda7 2024-11-25T10:58:00,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/9857d5243ab2461eafa78b5b4856aa0f is 50, key is test_row_0/C:col10/1732532278350/Put/seqid=0 2024-11-25T10:58:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T10:58:00,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742406_1582 (size=12301) 2024-11-25T10:58:00,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/9857d5243ab2461eafa78b5b4856aa0f 2024-11-25T10:58:00,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:00,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:00,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/32acdde9d5c44bc3a6dd687a553743dd as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/32acdde9d5c44bc3a6dd687a553743dd 2024-11-25T10:58:00,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/32acdde9d5c44bc3a6dd687a553743dd, entries=150, sequenceid=271, filesize=30.5 K 2024-11-25T10:58:00,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/b9734b5387074155b93266f64ac4dda7 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b9734b5387074155b93266f64ac4dda7 2024-11-25T10:58:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b9734b5387074155b93266f64ac4dda7, entries=150, sequenceid=271, filesize=12.0 K 2024-11-25T10:58:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/9857d5243ab2461eafa78b5b4856aa0f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/9857d5243ab2461eafa78b5b4856aa0f 2024-11-25T10:58:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/9857d5243ab2461eafa78b5b4856aa0f, entries=150, sequenceid=271, filesize=12.0 K 2024-11-25T10:58:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fb047475327d71fbf830ed8c3260c99d in 1201ms, sequenceid=271, compaction requested=true 2024-11-25T10:58:00,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,754 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:58:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:58:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:58:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:58:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:58:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:58:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:58:00,756 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:58:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,765 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:58:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,765 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/A is initiating minor compaction (all files) 2024-11-25T10:58:00,765 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/A in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,765 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/859038cb49424afeb3c0132b3a661430, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/070b9075308c40d59adedb86fff176d9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b1c93139d30a4cbda41f4b75b46f3b6f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/32acdde9d5c44bc3a6dd687a553743dd] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=139.0 K 2024-11-25T10:58:00,765 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,765 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/859038cb49424afeb3c0132b3a661430, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/070b9075308c40d59adedb86fff176d9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b1c93139d30a4cbda41f4b75b46f3b6f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/32acdde9d5c44bc3a6dd687a553743dd] 2024-11-25T10:58:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,769 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:58:00,769 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 859038cb49424afeb3c0132b3a661430, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732532275963 2024-11-25T10:58:00,769 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/B is initiating minor compaction (all files) 2024-11-25T10:58:00,769 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/B in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,769 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b85398a6dd4142e8b60793577192d8ca, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/26149a4921bc48b5aca48f1322cd07f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d0ebc15fe725469793ccdd87dcbe543d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b9734b5387074155b93266f64ac4dda7] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=48.1 K 2024-11-25T10:58:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,773 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b85398a6dd4142e8b60793577192d8ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732532275963 2024-11-25T10:58:00,773 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 070b9075308c40d59adedb86fff176d9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732532276066 2024-11-25T10:58:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,777 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26149a4921bc48b5aca48f1322cd07f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732532276076 2024-11-25T10:58:00,778 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c93139d30a4cbda41f4b75b46f3b6f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732532276125 2024-11-25T10:58:00,778 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ebc15fe725469793ccdd87dcbe543d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732532276125 2024-11-25T10:58:00,778 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 32acdde9d5c44bc3a6dd687a553743dd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732532278350 2024-11-25T10:58:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,780 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9734b5387074155b93266f64ac4dda7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732532278350 2024-11-25T10:58:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T10:58:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:58:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:58:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:58:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256c93d3eff0c044d79917bd87bcbd3e3e_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_1/A:col10/1732532279613/Put/seqid=0 2024-11-25T10:58:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,845 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:00,852 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#B#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:58:00,853 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/c249ec94ba4e4809926eee1a457abeac is 50, key is test_row_0/B:col10/1732532278350/Put/seqid=0 2024-11-25T10:58:00,861 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112549501a8417124a579c416574ae843089_fb047475327d71fbf830ed8c3260c99d store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:00,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:00,873 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112549501a8417124a579c416574ae843089_fb047475327d71fbf830ed8c3260c99d, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:00,873 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112549501a8417124a579c416574ae843089_fb047475327d71fbf830ed8c3260c99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T10:58:00,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742407_1583 (size=12454) 2024-11-25T10:58:00,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742408_1584 (size=12949) 2024-11-25T10:58:00,907 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/c249ec94ba4e4809926eee1a457abeac as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/c249ec94ba4e4809926eee1a457abeac 2024-11-25T10:58:00,913 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/B of fb047475327d71fbf830ed8c3260c99d into c249ec94ba4e4809926eee1a457abeac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:58:00,913 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:00,913 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/B, priority=12, startTime=1732532280755; duration=0sec 2024-11-25T10:58:00,913 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:58:00,913 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:B 2024-11-25T10:58:00,914 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:58:00,915 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:58:00,915 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/C is initiating minor compaction (all files) 2024-11-25T10:58:00,915 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/C in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:00,915 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1ea38e3768ba48e1a760130d4363e69b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/0b28cc75afa045dcb86ff9f8158e5a11, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4de33e495feb4a9da0fc24a292823629, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/9857d5243ab2461eafa78b5b4856aa0f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=48.1 K 2024-11-25T10:58:00,916 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea38e3768ba48e1a760130d4363e69b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732532275963 2024-11-25T10:58:00,917 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b28cc75afa045dcb86ff9f8158e5a11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732532276076 2024-11-25T10:58:00,917 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4de33e495feb4a9da0fc24a292823629, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732532276125 2024-11-25T10:58:00,917 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9857d5243ab2461eafa78b5b4856aa0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732532278350 2024-11-25T10:58:00,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742409_1585 (size=4469) 2024-11-25T10:58:00,929 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#A#compaction#490 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:58:00,930 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/152de69807054a5db9183894aca8ba41 is 175, key is test_row_0/A:col10/1732532278350/Put/seqid=0 2024-11-25T10:58:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532340916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:00,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532340921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:00,947 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#C#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:58:00,948 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/1b610205216f49a694344edad1ee25e0 is 50, key is test_row_0/C:col10/1732532278350/Put/seqid=0 2024-11-25T10:58:00,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742410_1586 (size=31903) 2024-11-25T10:58:00,972 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/152de69807054a5db9183894aca8ba41 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/152de69807054a5db9183894aca8ba41 2024-11-25T10:58:00,979 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/A of fb047475327d71fbf830ed8c3260c99d into 152de69807054a5db9183894aca8ba41(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:58:00,979 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:00,979 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/A, priority=12, startTime=1732532280754; duration=0sec 2024-11-25T10:58:00,979 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:58:00,979 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:A 2024-11-25T10:58:01,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742411_1587 (size=12949) 2024-11-25T10:58:01,016 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/1b610205216f49a694344edad1ee25e0 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1b610205216f49a694344edad1ee25e0 2024-11-25T10:58:01,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,029 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/C of fb047475327d71fbf830ed8c3260c99d into 1b610205216f49a694344edad1ee25e0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:58:01,029 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:01,029 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/C, priority=12, startTime=1732532280755; duration=0sec 2024-11-25T10:58:01,029 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:58:01,029 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:C 2024-11-25T10:58:01,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532341043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:01,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532341060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:01,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:01,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:01,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T10:58:01,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532341257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:01,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532341281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:01,302 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:01,333 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256c93d3eff0c044d79917bd87bcbd3e3e_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256c93d3eff0c044d79917bd87bcbd3e3e_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:01,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/26a8c13c1a5f456da0cbba6400fd534b, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:01,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:01,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/26a8c13c1a5f456da0cbba6400fd534b is 175, key is test_row_1/A:col10/1732532279613/Put/seqid=0 2024-11-25T10:58:01,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:01,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742412_1588 (size=31251) 2024-11-25T10:58:01,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/26a8c13c1a5f456da0cbba6400fd534b 2024-11-25T10:58:01,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/f0a9f7e525cd48ad9ff2b5445742d252 is 50, key is test_row_1/B:col10/1732532279613/Put/seqid=0 2024-11-25T10:58:01,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742413_1589 (size=9857) 2024-11-25T10:58:01,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532341568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:01,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532341600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:01,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:01,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:01,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T10:58:01,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:01,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/f0a9f7e525cd48ad9ff2b5445742d252 2024-11-25T10:58:01,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/f98addbf21a24ee182e393467d18130c is 50, key is test_row_1/C:col10/1732532279613/Put/seqid=0 2024-11-25T10:58:01,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742414_1590 (size=9857) 2024-11-25T10:58:01,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/f98addbf21a24ee182e393467d18130c 2024-11-25T10:58:01,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:01,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:01,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:01,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T10:58:01,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/26a8c13c1a5f456da0cbba6400fd534b as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/26a8c13c1a5f456da0cbba6400fd534b 2024-11-25T10:58:02,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1732532341997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,004 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_2' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:58:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:02,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/26a8c13c1a5f456da0cbba6400fd534b, entries=150, sequenceid=286, filesize=30.5 K 2024-11-25T10:58:02,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/f0a9f7e525cd48ad9ff2b5445742d252 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f0a9f7e525cd48ad9ff2b5445742d252 2024-11-25T10:58:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53882 deadline: 1732532342057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,063 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:58:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:02,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f0a9f7e525cd48ad9ff2b5445742d252, entries=100, sequenceid=286, filesize=9.6 K 2024-11-25T10:58:02,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/f98addbf21a24ee182e393467d18130c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f98addbf21a24ee182e393467d18130c 2024-11-25T10:58:02,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53902 deadline: 1732532342074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,083 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18306 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:58:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532342083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:02,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f98addbf21a24ee182e393467d18130c, entries=100, sequenceid=286, filesize=9.6 K 2024-11-25T10:58:02,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for fb047475327d71fbf830ed8c3260c99d in 1314ms, sequenceid=286, compaction requested=false 2024-11-25T10:58:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:02,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T10:58:02,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:02,133 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:58:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:58:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:58:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:58:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:02,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. as already flushing 2024-11-25T10:58:02,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258be9b3fadf1c4781afb5a29e9cc5bead_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532280914/Put/seqid=0 2024-11-25T10:58:02,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742415_1591 (size=14994) 2024-11-25T10:58:02,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:02,238 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258be9b3fadf1c4781afb5a29e9cc5bead_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258be9b3fadf1c4781afb5a29e9cc5bead_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:02,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/cb180f9ffd374e459c2427fb28a68da4, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:02,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/cb180f9ffd374e459c2427fb28a68da4 is 175, key is test_row_0/A:col10/1732532280914/Put/seqid=0 2024-11-25T10:58:02,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742416_1592 (size=39949) 2024-11-25T10:58:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532342291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:02,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532342415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:02,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532342634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:02,681 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/cb180f9ffd374e459c2427fb28a68da4 2024-11-25T10:58:02,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/faeea76ab2c44f72b27f464399e2ce1c is 50, key is test_row_0/B:col10/1732532280914/Put/seqid=0 2024-11-25T10:58:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T10:58:02,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742417_1593 (size=12301) 2024-11-25T10:58:02,717 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/faeea76ab2c44f72b27f464399e2ce1c 2024-11-25T10:58:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a2b6ad868fe24c968d3de590052a0bcc is 50, key is test_row_0/C:col10/1732532280914/Put/seqid=0 2024-11-25T10:58:02,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742418_1594 (size=12301) 2024-11-25T10:58:02,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53904 deadline: 1732532342943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T10:58:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44671 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53878 deadline: 1732532343105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:03,112 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6bfc37bad115,44671,1732532110409 for 'test_row_0' in fb047475327d71fbf830ed8c3260c99d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T10:58:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 , the old value is region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T10:58:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., hostname=6bfc37bad115,44671,1732532110409, seqNum=5 because the exception is null or not the one we care about 2024-11-25T10:58:03,188 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a2b6ad868fe24c968d3de590052a0bcc 2024-11-25T10:58:03,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/cb180f9ffd374e459c2427fb28a68da4 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/cb180f9ffd374e459c2427fb28a68da4 2024-11-25T10:58:03,199 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/cb180f9ffd374e459c2427fb28a68da4, entries=200, sequenceid=308, filesize=39.0 K 2024-11-25T10:58:03,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/faeea76ab2c44f72b27f464399e2ce1c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/faeea76ab2c44f72b27f464399e2ce1c 2024-11-25T10:58:03,206 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/faeea76ab2c44f72b27f464399e2ce1c, entries=150, sequenceid=308, filesize=12.0 K 2024-11-25T10:58:03,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a2b6ad868fe24c968d3de590052a0bcc as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a2b6ad868fe24c968d3de590052a0bcc 2024-11-25T10:58:03,233 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a2b6ad868fe24c968d3de590052a0bcc, entries=150, sequenceid=308, filesize=12.0 K 2024-11-25T10:58:03,234 INFO [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fb047475327d71fbf830ed8c3260c99d in 1101ms, sequenceid=308, compaction requested=true 2024-11-25T10:58:03,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:03,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:03,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-25T10:58:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-25T10:58:03,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-25T10:58:03,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6860 sec 2024-11-25T10:58:03,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.6930 sec 2024-11-25T10:58:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:03,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T10:58:03,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:58:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:58:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:58:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:03,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b492b105b68c42f6b2528f1b419c9323_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532283466/Put/seqid=0 2024-11-25T10:58:03,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742419_1595 (size=14994) 2024-11-25T10:58:03,713 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-11-25T10:58:03,713 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:03,713 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,714 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:03,716 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-25T10:58:03,716 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:03,716 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-11-25T10:58:03,716 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,716 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:03,717 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,717 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,717 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:03,719 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-11-25T10:58:03,719 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:03,719 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,719 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:03,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:03,723 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-25T10:58:03,723 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:03,723 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,723 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:03,725 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-25T10:58:03,725 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:03,725 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,725 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:03,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:03,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:03,976 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b492b105b68c42f6b2528f1b419c9323_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b492b105b68c42f6b2528f1b419c9323_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:03,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/f6c4c4360aee49aeb3cd1bbda182a290, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:03,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/f6c4c4360aee49aeb3cd1bbda182a290 is 175, key is test_row_0/A:col10/1732532283466/Put/seqid=0 2024-11-25T10:58:03,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742420_1596 (size=39949) 2024-11-25T10:58:04,383 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/f6c4c4360aee49aeb3cd1bbda182a290 2024-11-25T10:58:04,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/0e2610e2dfca4542a118ef7fa911874f is 50, key is test_row_0/B:col10/1732532283466/Put/seqid=0 2024-11-25T10:58:04,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742421_1597 (size=12301) 2024-11-25T10:58:04,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/0e2610e2dfca4542a118ef7fa911874f 2024-11-25T10:58:04,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a4587ad46c13430f9b3318a94a8c316c is 50, key is test_row_0/C:col10/1732532283466/Put/seqid=0 2024-11-25T10:58:04,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742422_1598 (size=12301) 2024-11-25T10:58:04,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a4587ad46c13430f9b3318a94a8c316c 2024-11-25T10:58:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/f6c4c4360aee49aeb3cd1bbda182a290 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/f6c4c4360aee49aeb3cd1bbda182a290 2024-11-25T10:58:04,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/f6c4c4360aee49aeb3cd1bbda182a290, entries=200, sequenceid=322, filesize=39.0 K 2024-11-25T10:58:04,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/0e2610e2dfca4542a118ef7fa911874f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/0e2610e2dfca4542a118ef7fa911874f 2024-11-25T10:58:04,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/0e2610e2dfca4542a118ef7fa911874f, entries=150, sequenceid=322, filesize=12.0 K 2024-11-25T10:58:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a4587ad46c13430f9b3318a94a8c316c as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a4587ad46c13430f9b3318a94a8c316c 2024-11-25T10:58:04,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a4587ad46c13430f9b3318a94a8c316c, entries=150, sequenceid=322, filesize=12.0 K 2024-11-25T10:58:04,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=93.93 KB/96180 for fb047475327d71fbf830ed8c3260c99d in 972ms, sequenceid=322, compaction requested=true 2024-11-25T10:58:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T10:58:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:58:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T10:58:04,449 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:58:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:58:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb047475327d71fbf830ed8c3260c99d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T10:58:04,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T10:58:04,450 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:58:04,451 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/A is initiating minor compaction (all files) 2024-11-25T10:58:04,451 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/A in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:04,451 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/152de69807054a5db9183894aca8ba41, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/26a8c13c1a5f456da0cbba6400fd534b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/cb180f9ffd374e459c2427fb28a68da4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/f6c4c4360aee49aeb3cd1bbda182a290] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=139.7 K 2024-11-25T10:58:04,451 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:04,451 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. files: [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/152de69807054a5db9183894aca8ba41, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/26a8c13c1a5f456da0cbba6400fd534b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/cb180f9ffd374e459c2427fb28a68da4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/f6c4c4360aee49aeb3cd1bbda182a290] 2024-11-25T10:58:04,452 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:58:04,457 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152de69807054a5db9183894aca8ba41, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732532278350 2024-11-25T10:58:04,457 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26a8c13c1a5f456da0cbba6400fd534b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732532279613 2024-11-25T10:58:04,461 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb180f9ffd374e459c2427fb28a68da4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732532280896 2024-11-25T10:58:04,461 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:58:04,461 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/B is initiating minor compaction (all files) 2024-11-25T10:58:04,461 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/B in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:04,461 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/c249ec94ba4e4809926eee1a457abeac, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f0a9f7e525cd48ad9ff2b5445742d252, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/faeea76ab2c44f72b27f464399e2ce1c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/0e2610e2dfca4542a118ef7fa911874f] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=46.3 K 2024-11-25T10:58:04,461 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c4c4360aee49aeb3cd1bbda182a290, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732532282170 2024-11-25T10:58:04,461 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting c249ec94ba4e4809926eee1a457abeac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732532278350 2024-11-25T10:58:04,462 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a9f7e525cd48ad9ff2b5445742d252, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732532279613 2024-11-25T10:58:04,462 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting faeea76ab2c44f72b27f464399e2ce1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732532280912 2024-11-25T10:58:04,463 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e2610e2dfca4542a118ef7fa911874f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732532282222 2024-11-25T10:58:04,476 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:58:04,477 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/87cd92737fb7417488531786516b408d is 50, key is test_row_0/B:col10/1732532283466/Put/seqid=0 2024-11-25T10:58:04,477 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:04,486 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411255792ffba898441988374cfd54d19747c_fb047475327d71fbf830ed8c3260c99d store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:04,495 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411255792ffba898441988374cfd54d19747c_fb047475327d71fbf830ed8c3260c99d, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:04,496 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255792ffba898441988374cfd54d19747c_fb047475327d71fbf830ed8c3260c99d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742423_1599 (size=13085) 2024-11-25T10:58:04,524 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/87cd92737fb7417488531786516b408d as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/87cd92737fb7417488531786516b408d 2024-11-25T10:58:04,530 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/B of fb047475327d71fbf830ed8c3260c99d into 87cd92737fb7417488531786516b408d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:58:04,530 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:04,530 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/B, priority=12, startTime=1732532284449; duration=0sec 2024-11-25T10:58:04,530 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T10:58:04,530 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:B 2024-11-25T10:58:04,531 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T10:58:04,532 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T10:58:04,532 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1541): fb047475327d71fbf830ed8c3260c99d/C is initiating minor compaction (all files) 2024-11-25T10:58:04,532 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb047475327d71fbf830ed8c3260c99d/C in TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:04,532 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1b610205216f49a694344edad1ee25e0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f98addbf21a24ee182e393467d18130c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a2b6ad868fe24c968d3de590052a0bcc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a4587ad46c13430f9b3318a94a8c316c] into tmpdir=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp, totalSize=46.3 K 2024-11-25T10:58:04,532 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b610205216f49a694344edad1ee25e0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732532278350 2024-11-25T10:58:04,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting f98addbf21a24ee182e393467d18130c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732532279613 2024-11-25T10:58:04,533 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b6ad868fe24c968d3de590052a0bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732532280912 2024-11-25T10:58:04,534 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] compactions.Compactor(225): Compacting a4587ad46c13430f9b3318a94a8c316c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732532282222 2024-11-25T10:58:04,546 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#C#compaction#503 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:58:04,547 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/4416730ed95843d4b262041a042659f9 is 50, key is test_row_0/C:col10/1732532283466/Put/seqid=0 2024-11-25T10:58:04,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742424_1600 (size=4469) 2024-11-25T10:58:04,550 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb047475327d71fbf830ed8c3260c99d#A#compaction#502 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T10:58:04,550 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/3ddcea58223445e59064e931750296fb is 175, key is test_row_0/A:col10/1732532283466/Put/seqid=0 2024-11-25T10:58:04,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742426_1602 (size=32039) 2024-11-25T10:58:04,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742425_1601 (size=13085) 2024-11-25T10:58:04,564 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/3ddcea58223445e59064e931750296fb as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/3ddcea58223445e59064e931750296fb 2024-11-25T10:58:04,569 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/A of fb047475327d71fbf830ed8c3260c99d into 3ddcea58223445e59064e931750296fb(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:58:04,569 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:04,569 INFO [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/A, priority=12, startTime=1732532284449; duration=0sec 2024-11-25T10:58:04,570 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:58:04,570 DEBUG [RS:0;6bfc37bad115:44671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:A 2024-11-25T10:58:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T10:58:04,714 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:58:04,967 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/4416730ed95843d4b262041a042659f9 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4416730ed95843d4b262041a042659f9 2024-11-25T10:58:04,971 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fb047475327d71fbf830ed8c3260c99d/C of fb047475327d71fbf830ed8c3260c99d into 4416730ed95843d4b262041a042659f9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T10:58:04,971 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:04,971 INFO [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., storeName=fb047475327d71fbf830ed8c3260c99d/C, priority=12, startTime=1732532284449; duration=0sec 2024-11-25T10:58:04,971 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T10:58:04,971 DEBUG [RS:0;6bfc37bad115:44671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb047475327d71fbf830ed8c3260c99d:C 2024-11-25T10:58:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44671 {}] regionserver.HRegion(8855): Flush requested on fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:05,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T10:58:05,136 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-11-25T10:58:05,136 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:58:05,136 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:58:05,136 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:58:05,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:05,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:05,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112537b1dbabcc534f02b6ec14d2622bdd9d_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532285134/Put/seqid=0 2024-11-25T10:58:05,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742427_1603 (size=12454) 2024-11-25T10:58:05,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:05,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112537b1dbabcc534f02b6ec14d2622bdd9d_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112537b1dbabcc534f02b6ec14d2622bdd9d_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:05,168 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/2a634351a6eb41aaa08c00fe945d20f1, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:05,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/2a634351a6eb41aaa08c00fe945d20f1 is 175, key is test_row_0/A:col10/1732532285134/Put/seqid=0 2024-11-25T10:58:05,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742428_1604 (size=31255) 2024-11-25T10:58:05,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/2a634351a6eb41aaa08c00fe945d20f1 2024-11-25T10:58:05,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/48c054e9ff0144cea24e4f99e2dca6da is 50, key is test_row_0/B:col10/1732532285134/Put/seqid=0 2024-11-25T10:58:05,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742429_1605 (size=12301) 2024-11-25T10:58:05,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/48c054e9ff0144cea24e4f99e2dca6da 2024-11-25T10:58:06,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/19cbc5575a9d4dbb98ddeb6178aff3af is 50, key is test_row_0/C:col10/1732532285134/Put/seqid=0 2024-11-25T10:58:06,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742430_1606 (size=12301) 2024-11-25T10:58:06,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/19cbc5575a9d4dbb98ddeb6178aff3af 2024-11-25T10:58:06,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/2a634351a6eb41aaa08c00fe945d20f1 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2a634351a6eb41aaa08c00fe945d20f1 2024-11-25T10:58:06,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2a634351a6eb41aaa08c00fe945d20f1, entries=150, sequenceid=343, filesize=30.5 K 2024-11-25T10:58:06,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/48c054e9ff0144cea24e4f99e2dca6da as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/48c054e9ff0144cea24e4f99e2dca6da 2024-11-25T10:58:06,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/48c054e9ff0144cea24e4f99e2dca6da, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T10:58:06,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/19cbc5575a9d4dbb98ddeb6178aff3af as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/19cbc5575a9d4dbb98ddeb6178aff3af 2024-11-25T10:58:06,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/19cbc5575a9d4dbb98ddeb6178aff3af, entries=150, sequenceid=343, filesize=12.0 K 2024-11-25T10:58:06,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for fb047475327d71fbf830ed8c3260c99d in 1289ms, sequenceid=343, compaction requested=false 2024-11-25T10:58:06,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb047475327d71fbf830ed8c3260c99d: 2024-11-25T10:58:07,468 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T10:58:12,036 INFO [Thread-2274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2274. 2024-11-25T10:58:12,036 DEBUG [Thread-2274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:12,036 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,036 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:12,085 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-11-25T10:58:12,085 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:12,085 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,085 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:12,116 INFO [Thread-2272 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2272. 2024-11-25T10:58:12,116 DEBUG [Thread-2272 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T10:58:12,116 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,116 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2444 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2432 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2514 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2448 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2387 2024-11-25T10:58:12,116 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T10:58:12,117 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:58:12,117 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:58:12,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:12,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:12,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T10:58:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T10:58:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T10:58:12,121 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532292121"}]},"ts":"1732532292121"} 2024-11-25T10:58:12,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T10:58:12,123 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T10:58:12,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T10:58:12,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, UNASSIGN}] 2024-11-25T10:58:12,126 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, UNASSIGN 2024-11-25T10:58:12,127 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=CLOSING, regionLocation=6bfc37bad115,44671,1732532110409 2024-11-25T10:58:12,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, UNASSIGN because future has completed 2024-11-25T10:58:12,129 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T10:58:12,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409}] 2024-11-25T10:58:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T10:58:12,282 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(122): Close fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1722): Closing fb047475327d71fbf830ed8c3260c99d, disabling compactions & flushes 2024-11-25T10:58:12,283 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. after waiting 0 ms 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:12,283 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(2902): Flushing fb047475327d71fbf830ed8c3260c99d 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=A 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=B 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fb047475327d71fbf830ed8c3260c99d, store=C 2024-11-25T10:58:12,283 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T10:58:12,289 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125703d4ad68b904841ac8952ea090305f7_fb047475327d71fbf830ed8c3260c99d is 50, key is test_row_0/A:col10/1732532292114/Put/seqid=0 2024-11-25T10:58:12,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742431_1607 (size=9914) 2024-11-25T10:58:12,293 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T10:58:12,298 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125703d4ad68b904841ac8952ea090305f7_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125703d4ad68b904841ac8952ea090305f7_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:12,299 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/134e8b0464ac4e08b7b2973193e5e67f, store: [table=TestAcidGuarantees family=A region=fb047475327d71fbf830ed8c3260c99d] 2024-11-25T10:58:12,300 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/134e8b0464ac4e08b7b2973193e5e67f is 175, key is test_row_0/A:col10/1732532292114/Put/seqid=0 2024-11-25T10:58:12,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742432_1608 (size=22561) 2024-11-25T10:58:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T10:58:12,706 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/134e8b0464ac4e08b7b2973193e5e67f 2024-11-25T10:58:12,714 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/77f68a317d55408d83c00fa96dcde847 is 50, key is test_row_0/B:col10/1732532292114/Put/seqid=0 2024-11-25T10:58:12,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742433_1609 (size=9857) 2024-11-25T10:58:12,718 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/77f68a317d55408d83c00fa96dcde847 2024-11-25T10:58:12,729 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a138a15aa92c458b9df6697a5aabad1a is 50, key is test_row_0/C:col10/1732532292114/Put/seqid=0 2024-11-25T10:58:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T10:58:12,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742434_1610 (size=9857) 2024-11-25T10:58:13,147 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a138a15aa92c458b9df6697a5aabad1a 2024-11-25T10:58:13,152 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/A/134e8b0464ac4e08b7b2973193e5e67f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/134e8b0464ac4e08b7b2973193e5e67f 2024-11-25T10:58:13,156 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/134e8b0464ac4e08b7b2973193e5e67f, entries=100, sequenceid=349, filesize=22.0 K 2024-11-25T10:58:13,156 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/B/77f68a317d55408d83c00fa96dcde847 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/77f68a317d55408d83c00fa96dcde847 2024-11-25T10:58:13,160 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/77f68a317d55408d83c00fa96dcde847, entries=100, sequenceid=349, filesize=9.6 K 2024-11-25T10:58:13,161 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/.tmp/C/a138a15aa92c458b9df6697a5aabad1a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a138a15aa92c458b9df6697a5aabad1a 2024-11-25T10:58:13,165 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a138a15aa92c458b9df6697a5aabad1a, entries=100, sequenceid=349, filesize=9.6 K 2024-11-25T10:58:13,166 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fb047475327d71fbf830ed8c3260c99d in 882ms, sequenceid=349, compaction requested=true 2024-11-25T10:58:13,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9a3d96a748334a7583960bb42cfe5116, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/85e8084c03b342d9a50387878b9a77e8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2c2183bcaf1649aabc846bbb7acc10bf, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/071de5ca7e2e4e95a9ba5033aa631e1f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/60ee3a32a5ca4191b6b43962747a026e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/51a08cf6b7b84b249997f56d542fa704, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c077f4e6483d42c296ad4d0b804bfdcb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0f788aa87b204ba8b6c6ac8c288ae8d6, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/585bd6dfa84d4cfeb709d11f21bc2c9f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/af893bccba564b87885a52012525ef71, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9b9119034a374b4e97ae58111d32681e, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b96b4070b770456abcc24054eb37c3a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c153105448d94c3c88093c94096613a3, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0c4a2f4a198445d9afd414e29a8b2603, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b51c144bc3b940e08dc8ff271936cf60, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/859038cb49424afeb3c0132b3a661430, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/070b9075308c40d59adedb86fff176d9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b1c93139d30a4cbda41f4b75b46f3b6f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/152de69807054a5db9183894aca8ba41, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/32acdde9d5c44bc3a6dd687a553743dd, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/26a8c13c1a5f456da0cbba6400fd534b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/cb180f9ffd374e459c2427fb28a68da4, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/f6c4c4360aee49aeb3cd1bbda182a290] to archive 2024-11-25T10:58:13,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:58:13,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9a3d96a748334a7583960bb42cfe5116 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9a3d96a748334a7583960bb42cfe5116 2024-11-25T10:58:13,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/85e8084c03b342d9a50387878b9a77e8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/85e8084c03b342d9a50387878b9a77e8 2024-11-25T10:58:13,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2c2183bcaf1649aabc846bbb7acc10bf to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2c2183bcaf1649aabc846bbb7acc10bf 2024-11-25T10:58:13,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/071de5ca7e2e4e95a9ba5033aa631e1f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/071de5ca7e2e4e95a9ba5033aa631e1f 2024-11-25T10:58:13,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/60ee3a32a5ca4191b6b43962747a026e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/60ee3a32a5ca4191b6b43962747a026e 2024-11-25T10:58:13,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/51a08cf6b7b84b249997f56d542fa704 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/51a08cf6b7b84b249997f56d542fa704 2024-11-25T10:58:13,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c077f4e6483d42c296ad4d0b804bfdcb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c077f4e6483d42c296ad4d0b804bfdcb 2024-11-25T10:58:13,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0f788aa87b204ba8b6c6ac8c288ae8d6 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0f788aa87b204ba8b6c6ac8c288ae8d6 2024-11-25T10:58:13,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/585bd6dfa84d4cfeb709d11f21bc2c9f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/585bd6dfa84d4cfeb709d11f21bc2c9f 2024-11-25T10:58:13,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/af893bccba564b87885a52012525ef71 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/af893bccba564b87885a52012525ef71 2024-11-25T10:58:13,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9b9119034a374b4e97ae58111d32681e to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/9b9119034a374b4e97ae58111d32681e 2024-11-25T10:58:13,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b96b4070b770456abcc24054eb37c3a3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b96b4070b770456abcc24054eb37c3a3 2024-11-25T10:58:13,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c153105448d94c3c88093c94096613a3 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/c153105448d94c3c88093c94096613a3 2024-11-25T10:58:13,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0c4a2f4a198445d9afd414e29a8b2603 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/0c4a2f4a198445d9afd414e29a8b2603 2024-11-25T10:58:13,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b51c144bc3b940e08dc8ff271936cf60 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b51c144bc3b940e08dc8ff271936cf60 2024-11-25T10:58:13,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/859038cb49424afeb3c0132b3a661430 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/859038cb49424afeb3c0132b3a661430 2024-11-25T10:58:13,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/070b9075308c40d59adedb86fff176d9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/070b9075308c40d59adedb86fff176d9 2024-11-25T10:58:13,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b1c93139d30a4cbda41f4b75b46f3b6f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/b1c93139d30a4cbda41f4b75b46f3b6f 2024-11-25T10:58:13,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/152de69807054a5db9183894aca8ba41 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/152de69807054a5db9183894aca8ba41 2024-11-25T10:58:13,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/32acdde9d5c44bc3a6dd687a553743dd to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/32acdde9d5c44bc3a6dd687a553743dd 2024-11-25T10:58:13,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/26a8c13c1a5f456da0cbba6400fd534b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/26a8c13c1a5f456da0cbba6400fd534b 2024-11-25T10:58:13,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/cb180f9ffd374e459c2427fb28a68da4 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/cb180f9ffd374e459c2427fb28a68da4 2024-11-25T10:58:13,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/f6c4c4360aee49aeb3cd1bbda182a290 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/f6c4c4360aee49aeb3cd1bbda182a290 2024-11-25T10:58:13,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f506355ec43c4d769a48715bffea1185, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/1f3196c3eda64cdfb134e6ff6c9ca019, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/650935cde14d4f32829dc36561df9cb9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c80a3ee340d4dee97ee2d0a1012fc00, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/bd89b05faf314dc2a9b67955977d88c0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/176f8eda55864d98a023ed09d871cc40, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/2f6f75e9e420487884d2e6e1ca85cbb1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b6becf6a2166473ea543281a47819515, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/8c463cb8baf64aebbd18aac20fc92bf9, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/98ae5dc527ff480a99d932aa02eb51cb, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/56ce5d837c7f49ba9ea504b85e8bea52, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/e6e5b085710644848ef99d542baaa55c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c5310a5d5014a43a01994f17197c07c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d8f1d8aeaa614087899e04ce6d860da5, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b85398a6dd4142e8b60793577192d8ca, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/049994888a7e4b348b47355545485c9d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/26149a4921bc48b5aca48f1322cd07f7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d0ebc15fe725469793ccdd87dcbe543d, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/c249ec94ba4e4809926eee1a457abeac, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b9734b5387074155b93266f64ac4dda7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f0a9f7e525cd48ad9ff2b5445742d252, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/faeea76ab2c44f72b27f464399e2ce1c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/0e2610e2dfca4542a118ef7fa911874f] to archive 2024-11-25T10:58:13,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:58:13,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f506355ec43c4d769a48715bffea1185 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f506355ec43c4d769a48715bffea1185 2024-11-25T10:58:13,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/1f3196c3eda64cdfb134e6ff6c9ca019 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/1f3196c3eda64cdfb134e6ff6c9ca019 2024-11-25T10:58:13,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/650935cde14d4f32829dc36561df9cb9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/650935cde14d4f32829dc36561df9cb9 2024-11-25T10:58:13,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c80a3ee340d4dee97ee2d0a1012fc00 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c80a3ee340d4dee97ee2d0a1012fc00 2024-11-25T10:58:13,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/bd89b05faf314dc2a9b67955977d88c0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/bd89b05faf314dc2a9b67955977d88c0 2024-11-25T10:58:13,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/176f8eda55864d98a023ed09d871cc40 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/176f8eda55864d98a023ed09d871cc40 2024-11-25T10:58:13,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/2f6f75e9e420487884d2e6e1ca85cbb1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/2f6f75e9e420487884d2e6e1ca85cbb1 2024-11-25T10:58:13,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b6becf6a2166473ea543281a47819515 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b6becf6a2166473ea543281a47819515 2024-11-25T10:58:13,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/8c463cb8baf64aebbd18aac20fc92bf9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/8c463cb8baf64aebbd18aac20fc92bf9 2024-11-25T10:58:13,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/98ae5dc527ff480a99d932aa02eb51cb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/98ae5dc527ff480a99d932aa02eb51cb 2024-11-25T10:58:13,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/56ce5d837c7f49ba9ea504b85e8bea52 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/56ce5d837c7f49ba9ea504b85e8bea52 2024-11-25T10:58:13,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/e6e5b085710644848ef99d542baaa55c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/e6e5b085710644848ef99d542baaa55c 2024-11-25T10:58:13,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c5310a5d5014a43a01994f17197c07c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/6c5310a5d5014a43a01994f17197c07c 2024-11-25T10:58:13,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d8f1d8aeaa614087899e04ce6d860da5 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d8f1d8aeaa614087899e04ce6d860da5 2024-11-25T10:58:13,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b85398a6dd4142e8b60793577192d8ca to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b85398a6dd4142e8b60793577192d8ca 2024-11-25T10:58:13,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/049994888a7e4b348b47355545485c9d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/049994888a7e4b348b47355545485c9d 2024-11-25T10:58:13,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/26149a4921bc48b5aca48f1322cd07f7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/26149a4921bc48b5aca48f1322cd07f7 2024-11-25T10:58:13,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d0ebc15fe725469793ccdd87dcbe543d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/d0ebc15fe725469793ccdd87dcbe543d 2024-11-25T10:58:13,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/c249ec94ba4e4809926eee1a457abeac to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/c249ec94ba4e4809926eee1a457abeac 2024-11-25T10:58:13,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b9734b5387074155b93266f64ac4dda7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/b9734b5387074155b93266f64ac4dda7 2024-11-25T10:58:13,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f0a9f7e525cd48ad9ff2b5445742d252 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/f0a9f7e525cd48ad9ff2b5445742d252 2024-11-25T10:58:13,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/faeea76ab2c44f72b27f464399e2ce1c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/faeea76ab2c44f72b27f464399e2ce1c 2024-11-25T10:58:13,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/0e2610e2dfca4542a118ef7fa911874f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/0e2610e2dfca4542a118ef7fa911874f 2024-11-25T10:58:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T10:58:13,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/3537ef0f8ab7480680499293b016e861, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f9d4f202fc0d47be811cd56eadd934f8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1c78df60493740d79698081e189d6ece, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/6d18fa7b678f4bc293cc29b2a6545212, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/457b0854ead74ca292d4ed80ec1a952c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4e8567dca2b1441fb5842775027025ec, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/e22b355e35e94b38b175fded7348c798, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/d31b1361bb7749b6ab022696b7c0c2b1, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/77d562cf4ada4ef08fe097063d61de56, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a10bb8d525f74be4b6cbea04c7a35242, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/5f9e7978e0644af6a5083d842b006d89, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/506a06319ece45759247ff29d814b008, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/355ea117afcb44f6ac8849ec4523b716, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/c825c936d9fc41e2b99635df5be9aae7, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1ea38e3768ba48e1a760130d4363e69b, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/332d4f7cd5ef4a3b9af49216fc6b50d8, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/0b28cc75afa045dcb86ff9f8158e5a11, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4de33e495feb4a9da0fc24a292823629, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1b610205216f49a694344edad1ee25e0, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/9857d5243ab2461eafa78b5b4856aa0f, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f98addbf21a24ee182e393467d18130c, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a2b6ad868fe24c968d3de590052a0bcc, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a4587ad46c13430f9b3318a94a8c316c] to archive 2024-11-25T10:58:13,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T10:58:13,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/3537ef0f8ab7480680499293b016e861 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/3537ef0f8ab7480680499293b016e861 2024-11-25T10:58:13,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f9d4f202fc0d47be811cd56eadd934f8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f9d4f202fc0d47be811cd56eadd934f8 2024-11-25T10:58:13,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1c78df60493740d79698081e189d6ece to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1c78df60493740d79698081e189d6ece 2024-11-25T10:58:13,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/6d18fa7b678f4bc293cc29b2a6545212 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/6d18fa7b678f4bc293cc29b2a6545212 2024-11-25T10:58:13,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/457b0854ead74ca292d4ed80ec1a952c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/457b0854ead74ca292d4ed80ec1a952c 2024-11-25T10:58:13,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4e8567dca2b1441fb5842775027025ec to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4e8567dca2b1441fb5842775027025ec 2024-11-25T10:58:13,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/e22b355e35e94b38b175fded7348c798 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/e22b355e35e94b38b175fded7348c798 2024-11-25T10:58:13,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/d31b1361bb7749b6ab022696b7c0c2b1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/d31b1361bb7749b6ab022696b7c0c2b1 2024-11-25T10:58:13,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/77d562cf4ada4ef08fe097063d61de56 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/77d562cf4ada4ef08fe097063d61de56 2024-11-25T10:58:13,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a10bb8d525f74be4b6cbea04c7a35242 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a10bb8d525f74be4b6cbea04c7a35242 2024-11-25T10:58:13,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/5f9e7978e0644af6a5083d842b006d89 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/5f9e7978e0644af6a5083d842b006d89 2024-11-25T10:58:13,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/506a06319ece45759247ff29d814b008 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/506a06319ece45759247ff29d814b008 2024-11-25T10:58:13,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/355ea117afcb44f6ac8849ec4523b716 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/355ea117afcb44f6ac8849ec4523b716 2024-11-25T10:58:13,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/c825c936d9fc41e2b99635df5be9aae7 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/c825c936d9fc41e2b99635df5be9aae7 2024-11-25T10:58:13,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1ea38e3768ba48e1a760130d4363e69b to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1ea38e3768ba48e1a760130d4363e69b 2024-11-25T10:58:13,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/332d4f7cd5ef4a3b9af49216fc6b50d8 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/332d4f7cd5ef4a3b9af49216fc6b50d8 2024-11-25T10:58:13,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/0b28cc75afa045dcb86ff9f8158e5a11 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/0b28cc75afa045dcb86ff9f8158e5a11 2024-11-25T10:58:13,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4de33e495feb4a9da0fc24a292823629 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4de33e495feb4a9da0fc24a292823629 2024-11-25T10:58:13,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1b610205216f49a694344edad1ee25e0 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/1b610205216f49a694344edad1ee25e0 2024-11-25T10:58:13,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/9857d5243ab2461eafa78b5b4856aa0f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/9857d5243ab2461eafa78b5b4856aa0f 2024-11-25T10:58:13,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f98addbf21a24ee182e393467d18130c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/f98addbf21a24ee182e393467d18130c 2024-11-25T10:58:13,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a2b6ad868fe24c968d3de590052a0bcc to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a2b6ad868fe24c968d3de590052a0bcc 2024-11-25T10:58:13,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a4587ad46c13430f9b3318a94a8c316c to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a4587ad46c13430f9b3318a94a8c316c 2024-11-25T10:58:13,306 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/recovered.edits/352.seqid, newMaxSeqId=352, maxSeqId=4 2024-11-25T10:58:13,307 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. 2024-11-25T10:58:13,325 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1676): Region close journal for fb047475327d71fbf830ed8c3260c99d: Waiting for close lock at 1732532292283Running coprocessor pre-close hooks at 1732532292283Disabling compacts and flushes for region at 1732532292283Disabling writes for close at 1732532292283Obtaining lock to block concurrent updates at 1732532292283Preparing flush snapshotting stores in fb047475327d71fbf830ed8c3260c99d at 1732532292283Finished memstore snapshotting TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732532292283Flushing stores of TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d. at 1732532292284 (+1 ms)Flushing fb047475327d71fbf830ed8c3260c99d/A: creating writer at 1732532292284Flushing mob file fb047475327d71fbf830ed8c3260c99d/A: appending metadata at 1732532292289 (+5 ms)Flushing mob file fb047475327d71fbf830ed8c3260c99d/A: closing flushed file at 1732532292289Flushing fb047475327d71fbf830ed8c3260c99d/A: appending metadata at 1732532292299 (+10 ms)Flushing fb047475327d71fbf830ed8c3260c99d/A: closing flushed file at 1732532292299Flushing fb047475327d71fbf830ed8c3260c99d/B: creating writer at 1732532292710 (+411 ms)Flushing fb047475327d71fbf830ed8c3260c99d/B: appending metadata at 1732532292713 (+3 ms)Flushing fb047475327d71fbf830ed8c3260c99d/B: closing flushed file at 1732532292713Flushing fb047475327d71fbf830ed8c3260c99d/C: creating writer at 1732532292725 (+12 ms)Flushing fb047475327d71fbf830ed8c3260c99d/C: appending metadata at 1732532292728 (+3 ms)Flushing fb047475327d71fbf830ed8c3260c99d/C: closing flushed file at 1732532292728Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cfecffe: reopening flushed file at 1732532293151 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@205b989c: reopening flushed file at 1732532293156 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41ef02df: reopening flushed file at 1732532293160 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fb047475327d71fbf830ed8c3260c99d in 882ms, sequenceid=349, compaction requested=true at 1732532293166 (+6 ms)Writing region close event to WAL at 1732532293297 (+131 ms)Running coprocessor post-close hooks at 1732532293307 (+10 ms)Closed at 1732532293307 2024-11-25T10:58:13,337 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(157): Closed fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:13,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=fb047475327d71fbf830ed8c3260c99d, regionState=CLOSED 2024-11-25T10:58:13,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 because future has completed 2024-11-25T10:58:13,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-25T10:58:13,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseRegionProcedure fb047475327d71fbf830ed8c3260c99d, server=6bfc37bad115,44671,1732532110409 in 1.2310 sec 2024-11-25T10:58:13,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-25T10:58:13,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fb047475327d71fbf830ed8c3260c99d, UNASSIGN in 1.2480 sec 2024-11-25T10:58:13,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-25T10:58:13,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2620 sec 2024-11-25T10:58:13,389 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532293389"}]},"ts":"1732532293389"} 2024-11-25T10:58:13,397 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T10:58:13,397 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T10:58:13,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2800 sec 2024-11-25T10:58:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T10:58:14,263 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:58:14,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T10:58:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:58:14,266 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:58:14,267 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=156, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:58:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T10:58:14,269 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,273 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C, FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/recovered.edits] 2024-11-25T10:58:14,276 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/134e8b0464ac4e08b7b2973193e5e67f to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/134e8b0464ac4e08b7b2973193e5e67f 2024-11-25T10:58:14,277 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2a634351a6eb41aaa08c00fe945d20f1 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/2a634351a6eb41aaa08c00fe945d20f1 2024-11-25T10:58:14,281 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/3ddcea58223445e59064e931750296fb to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/A/3ddcea58223445e59064e931750296fb 2024-11-25T10:58:14,286 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/48c054e9ff0144cea24e4f99e2dca6da to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/48c054e9ff0144cea24e4f99e2dca6da 2024-11-25T10:58:14,287 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/77f68a317d55408d83c00fa96dcde847 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/77f68a317d55408d83c00fa96dcde847 2024-11-25T10:58:14,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/87cd92737fb7417488531786516b408d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/B/87cd92737fb7417488531786516b408d 2024-11-25T10:58:14,290 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/19cbc5575a9d4dbb98ddeb6178aff3af to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/19cbc5575a9d4dbb98ddeb6178aff3af 2024-11-25T10:58:14,291 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4416730ed95843d4b262041a042659f9 to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/4416730ed95843d4b262041a042659f9 2024-11-25T10:58:14,292 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a138a15aa92c458b9df6697a5aabad1a to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/C/a138a15aa92c458b9df6697a5aabad1a 2024-11-25T10:58:14,295 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/recovered.edits/352.seqid to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d/recovered.edits/352.seqid 2024-11-25T10:58:14,295 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/default/TestAcidGuarantees/fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,295 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T10:58:14,296 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T10:58:14,296 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T10:58:14,299 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250191517ed8dc4c59a7ccc1e8a38dd5fe_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250191517ed8dc4c59a7ccc1e8a38dd5fe_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,300 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250fe39edb834145f788b704de4f3be208_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250fe39edb834145f788b704de4f3be208_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,301 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251459eb2a215747d495f1d9fd1a97ed00_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251459eb2a215747d495f1d9fd1a97ed00_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,303 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112529b129ded9e84957ad788c0901064c06_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112529b129ded9e84957ad788c0901064c06_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,304 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252bc4c70060ec4464ba12e6abcd1a7781_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252bc4c70060ec4464ba12e6abcd1a7781_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,304 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112537b1dbabcc534f02b6ec14d2622bdd9d_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112537b1dbabcc534f02b6ec14d2622bdd9d_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,305 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256c93d3eff0c044d79917bd87bcbd3e3e_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256c93d3eff0c044d79917bd87bcbd3e3e_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,307 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125703d4ad68b904841ac8952ea090305f7_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125703d4ad68b904841ac8952ea090305f7_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,307 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258be9b3fadf1c4781afb5a29e9cc5bead_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258be9b3fadf1c4781afb5a29e9cc5bead_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,308 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259856dfcabaf04f18bd3878e7e43a9894_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259856dfcabaf04f18bd3878e7e43a9894_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,309 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259be5264a16fd4196801d086811c03fb9_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259be5264a16fd4196801d086811c03fb9_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,310 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259fe4d13d45e345ef888efec30c38b4ad_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259fe4d13d45e345ef888efec30c38b4ad_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,311 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a70b0e4b127f426bbe415cc900db6f3a_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a70b0e4b127f426bbe415cc900db6f3a_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,312 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b492b105b68c42f6b2528f1b419c9323_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b492b105b68c42f6b2528f1b419c9323_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,313 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b55c3e7c8eed4431b5b7b7ab93949081_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b55c3e7c8eed4431b5b7b7ab93949081_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,314 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bc86a2c198704c3fbe46faaf4238fffb_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bc86a2c198704c3fbe46faaf4238fffb_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,315 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bf56f1b564b14f17a5bd3cbf4f22bb4b_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bf56f1b564b14f17a5bd3cbf4f22bb4b_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,316 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c50431e45b574d3989a71dfc1e7cd925_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c50431e45b574d3989a71dfc1e7cd925_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,317 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f1f7873d966448f1a566535187e923b0_fb047475327d71fbf830ed8c3260c99d to hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f1f7873d966448f1a566535187e923b0_fb047475327d71fbf830ed8c3260c99d 2024-11-25T10:58:14,318 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T10:58:14,320 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=156, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:58:14,323 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T10:58:14,325 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T10:58:14,326 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=156, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:58:14,326 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T10:58:14,327 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732532294326"}]},"ts":"9223372036854775807"} 2024-11-25T10:58:14,328 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T10:58:14,328 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fb047475327d71fbf830ed8c3260c99d, NAME => 'TestAcidGuarantees,,1732532261004.fb047475327d71fbf830ed8c3260c99d.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T10:58:14,329 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T10:58:14,329 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732532294329"}]},"ts":"9223372036854775807"} 2024-11-25T10:58:14,330 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T10:58:14,331 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=156, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T10:58:14,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-25T10:58:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T10:58:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T10:58:14,379 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T10:58:14,397 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 247), OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1048 (was 1232), ProcessCount=11 (was 11), AvailableMemoryMB=1352 (was 2143) 2024-11-25T10:58:14,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T10:58:14,398 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:58:14,398 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:58:14,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:14,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:14,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T10:58:14,398 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T10:58:14,398 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1258311727, stopped=false 2024-11-25T10:58:14,399 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6bfc37bad115,39013,1732532109078 2024-11-25T10:58:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T10:58:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:58:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T10:58:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:58:14,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T10:58:14,402 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T10:58:14,402 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T10:58:14,403 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:58:14,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:14,403 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6bfc37bad115,44671,1732532110409' ***** 2024-11-25T10:58:14,403 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T10:58:14,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T10:58:14,404 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T10:58:14,404 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T10:58:14,404 INFO [RS:0;6bfc37bad115:44671 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T10:58:14,404 INFO [RS:0;6bfc37bad115:44671 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T10:58:14,404 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(959): stopping server 6bfc37bad115,44671,1732532110409 2024-11-25T10:58:14,404 INFO [RS:0;6bfc37bad115:44671 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T10:58:14,404 INFO [RS:0;6bfc37bad115:44671 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6bfc37bad115:44671. 2024-11-25T10:58:14,404 DEBUG [RS:0;6bfc37bad115:44671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T10:58:14,404 DEBUG [RS:0;6bfc37bad115:44671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:14,405 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T10:58:14,405 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T10:58:14,405 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T10:58:14,405 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T10:58:14,406 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T10:58:14,406 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T10:58:14,406 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T10:58:14,406 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T10:58:14,406 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T10:58:14,406 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T10:58:14,406 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T10:58:14,406 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T10:58:14,406 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-25T10:58:14,412 INFO [regionserver/6bfc37bad115:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T10:58:14,449 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/info/bb819db240714895a503d0d75702f0ca is 95, key is TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1./info:/1732532143627/DeleteFamily/seqid=0 2024-11-25T10:58:14,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742435_1611 (size=6514) 2024-11-25T10:58:14,501 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/info/bb819db240714895a503d0d75702f0ca 2024-11-25T10:58:14,540 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/ns/8b63cdd63cdc4533b621b13327f27499 is 93, key is TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1./ns:/1732532143520/DeleteFamily/seqid=0 2024-11-25T10:58:14,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742436_1612 (size=5906) 2024-11-25T10:58:14,558 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/ns/8b63cdd63cdc4533b621b13327f27499 2024-11-25T10:58:14,584 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/rep_barrier/34b47a6c29ad41d3882b299062d40e5a is 102, key is TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1./rep_barrier:/1732532143520/DeleteFamily/seqid=0 2024-11-25T10:58:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742437_1613 (size=6025) 2024-11-25T10:58:14,606 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T10:58:14,807 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T10:58:14,991 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/rep_barrier/34b47a6c29ad41d3882b299062d40e5a 2024-11-25T10:58:15,007 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T10:58:15,017 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/table/e136097d965c48fa826dda2c162d8090 is 96, key is TestAcidGuarantees,,1732532115481.619a47d3e2a0b4c567d3fd0e3f1893d1./table:/1732532143520/DeleteFamily/seqid=0 2024-11-25T10:58:15,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742438_1614 (size=5936) 2024-11-25T10:58:15,053 INFO [regionserver/6bfc37bad115:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T10:58:15,053 INFO [regionserver/6bfc37bad115:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T10:58:15,207 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T10:58:15,407 DEBUG [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T10:58:15,433 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/table/e136097d965c48fa826dda2c162d8090 2024-11-25T10:58:15,437 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/info/bb819db240714895a503d0d75702f0ca as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/info/bb819db240714895a503d0d75702f0ca 2024-11-25T10:58:15,441 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/info/bb819db240714895a503d0d75702f0ca, entries=12, sequenceid=90, filesize=6.4 K 2024-11-25T10:58:15,442 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/ns/8b63cdd63cdc4533b621b13327f27499 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/ns/8b63cdd63cdc4533b621b13327f27499 2024-11-25T10:58:15,445 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/ns/8b63cdd63cdc4533b621b13327f27499, entries=8, sequenceid=90, filesize=5.8 K 2024-11-25T10:58:15,446 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/rep_barrier/34b47a6c29ad41d3882b299062d40e5a as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/rep_barrier/34b47a6c29ad41d3882b299062d40e5a 2024-11-25T10:58:15,450 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/rep_barrier/34b47a6c29ad41d3882b299062d40e5a, entries=6, sequenceid=90, filesize=5.9 K 2024-11-25T10:58:15,450 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/.tmp/table/e136097d965c48fa826dda2c162d8090 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/table/e136097d965c48fa826dda2c162d8090 2024-11-25T10:58:15,454 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/table/e136097d965c48fa826dda2c162d8090, entries=7, sequenceid=90, filesize=5.8 K 2024-11-25T10:58:15,456 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1050ms, sequenceid=90, compaction requested=false 2024-11-25T10:58:15,466 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-25T10:58:15,467 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T10:58:15,467 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T10:58:15,467 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732532294406Running coprocessor pre-close hooks at 1732532294406Disabling compacts and flushes for region at 1732532294406Disabling writes for close at 1732532294406Obtaining lock to block concurrent updates at 1732532294406Preparing flush snapshotting stores in 1588230740 at 1732532294406Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732532294407 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732532294409 (+2 ms)Flushing 1588230740/info: creating writer at 1732532294409Flushing 1588230740/info: appending metadata at 1732532294448 (+39 ms)Flushing 1588230740/info: closing flushed file at 1732532294448Flushing 1588230740/ns: creating writer at 1732532294518 (+70 ms)Flushing 1588230740/ns: appending metadata at 1732532294539 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732532294539Flushing 1588230740/rep_barrier: creating writer at 1732532294562 (+23 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732532294583 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732532294583Flushing 1588230740/table: creating writer at 1732532294995 (+412 ms)Flushing 1588230740/table: appending metadata at 1732532295017 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732532295017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73eda586: reopening flushed file at 1732532295437 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1930f0d3: reopening flushed file at 1732532295441 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@662ba255: reopening flushed file at 1732532295445 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55c73714: reopening flushed file at 1732532295450 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1050ms, sequenceid=90, compaction requested=false at 1732532295456 (+6 ms)Writing region close event to WAL at 1732532295463 (+7 ms)Running coprocessor post-close hooks at 1732532295467 (+4 ms)Closed at 1732532295467 2024-11-25T10:58:15,467 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T10:58:15,608 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(976): stopping server 6bfc37bad115,44671,1732532110409; all regions closed. 2024-11-25T10:58:15,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741834_1010 (size=25344) 2024-11-25T10:58:15,617 DEBUG [RS:0;6bfc37bad115:44671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/oldWALs 2024-11-25T10:58:15,617 INFO [RS:0;6bfc37bad115:44671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C44671%2C1732532110409.meta:.meta(num 1732532114730) 2024-11-25T10:58:15,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741833_1009 (size=14694066) 2024-11-25T10:58:15,621 DEBUG [RS:0;6bfc37bad115:44671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/oldWALs 2024-11-25T10:58:15,621 INFO [RS:0;6bfc37bad115:44671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C44671%2C1732532110409:(num 1732532114199) 2024-11-25T10:58:15,621 DEBUG [RS:0;6bfc37bad115:44671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T10:58:15,621 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T10:58:15,621 INFO [RS:0;6bfc37bad115:44671 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T10:58:15,622 INFO [RS:0;6bfc37bad115:44671 {}] hbase.ChoreService(370): Chore service for: regionserver/6bfc37bad115:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T10:58:15,622 INFO [RS:0;6bfc37bad115:44671 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T10:58:15,622 INFO [regionserver/6bfc37bad115:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T10:58:15,623 INFO [RS:0;6bfc37bad115:44671 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44671 2024-11-25T10:58:15,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T10:58:15,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6bfc37bad115,44671,1732532110409 2024-11-25T10:58:15,629 INFO [RS:0;6bfc37bad115:44671 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T10:58:15,631 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6bfc37bad115,44671,1732532110409] 2024-11-25T10:58:15,634 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6bfc37bad115,44671,1732532110409 already deleted, retry=false 2024-11-25T10:58:15,635 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6bfc37bad115,44671,1732532110409 expired; onlineServers=0 2024-11-25T10:58:15,635 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6bfc37bad115,39013,1732532109078' ***** 2024-11-25T10:58:15,635 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T10:58:15,635 INFO [M:0;6bfc37bad115:39013 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T10:58:15,635 INFO [M:0;6bfc37bad115:39013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T10:58:15,635 DEBUG [M:0;6bfc37bad115:39013 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T10:58:15,635 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T10:58:15,635 DEBUG [M:0;6bfc37bad115:39013 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T10:58:15,635 DEBUG [master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532113646 {}] cleaner.HFileCleaner(306): Exit Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532113646,5,FailOnTimeoutGroup] 2024-11-25T10:58:15,635 DEBUG [master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532113664 {}] cleaner.HFileCleaner(306): Exit Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532113664,5,FailOnTimeoutGroup] 2024-11-25T10:58:15,636 INFO [M:0;6bfc37bad115:39013 {}] hbase.ChoreService(370): Chore service for: master/6bfc37bad115:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T10:58:15,636 INFO [M:0;6bfc37bad115:39013 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T10:58:15,636 DEBUG [M:0;6bfc37bad115:39013 {}] master.HMaster(1795): Stopping service threads 2024-11-25T10:58:15,636 INFO [M:0;6bfc37bad115:39013 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T10:58:15,636 INFO [M:0;6bfc37bad115:39013 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T10:58:15,636 ERROR [M:0;6bfc37bad115:39013 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-25T10:58:15,637 INFO [M:0;6bfc37bad115:39013 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T10:58:15,637 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T10:58:15,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T10:58:15,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T10:58:15,642 DEBUG [M:0;6bfc37bad115:39013 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-25T10:58:15,642 DEBUG [M:0;6bfc37bad115:39013 {}] master.ActiveMasterManager(353): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-25T10:58:15,643 INFO [M:0;6bfc37bad115:39013 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/.lastflushedseqids 2024-11-25T10:58:15,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742439_1615 (size=72) 2024-11-25T10:58:15,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T10:58:15,733 INFO [RS:0;6bfc37bad115:44671 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T10:58:15,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44671-0x10076940c1b0001, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T10:58:15,734 INFO [RS:0;6bfc37bad115:44671 {}] regionserver.HRegionServer(1031): Exiting; stopping=6bfc37bad115,44671,1732532110409; zookeeper connection closed. 2024-11-25T10:58:15,734 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2d147d19 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2d147d19 2024-11-25T10:58:15,735 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T10:58:16,061 INFO [M:0;6bfc37bad115:39013 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T10:58:16,061 INFO [M:0;6bfc37bad115:39013 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T10:58:16,061 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T10:58:16,061 INFO [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:58:16,061 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:58:16,061 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T10:58:16,061 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:58:16,061 INFO [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=773.33 KB heapSize=948.22 KB 2024-11-25T10:58:16,085 DEBUG [M:0;6bfc37bad115:39013 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f3e136c020f42079a12ee1473d4da31 is 82, key is hbase:meta,,1/info:regioninfo/1732532114888/Put/seqid=0 2024-11-25T10:58:16,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742440_1616 (size=5672) 2024-11-25T10:58:16,495 INFO [M:0;6bfc37bad115:39013 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2171 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f3e136c020f42079a12ee1473d4da31 2024-11-25T10:58:16,522 DEBUG [M:0;6bfc37bad115:39013 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/23def87852b949b08790e99c257e6cc2 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1732532147145/Put/seqid=0 2024-11-25T10:58:16,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742441_1617 (size=42906) 2024-11-25T10:58:16,540 INFO [M:0;6bfc37bad115:39013 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=772.77 KB at sequenceid=2171 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/23def87852b949b08790e99c257e6cc2 2024-11-25T10:58:16,547 INFO [M:0;6bfc37bad115:39013 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 23def87852b949b08790e99c257e6cc2 2024-11-25T10:58:16,568 DEBUG [M:0;6bfc37bad115:39013 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8df3eee21a2940d281883f6b86add61f is 69, key is 6bfc37bad115,44671,1732532110409/rs:state/1732532113797/Put/seqid=0 2024-11-25T10:58:16,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073742442_1618 (size=5156) 2024-11-25T10:58:16,572 INFO [M:0;6bfc37bad115:39013 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2171 (bloomFilter=true), to=hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8df3eee21a2940d281883f6b86add61f 2024-11-25T10:58:16,576 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f3e136c020f42079a12ee1473d4da31 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4f3e136c020f42079a12ee1473d4da31 2024-11-25T10:58:16,580 INFO [M:0;6bfc37bad115:39013 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4f3e136c020f42079a12ee1473d4da31, entries=8, sequenceid=2171, filesize=5.5 K 2024-11-25T10:58:16,580 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/23def87852b949b08790e99c257e6cc2 as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/23def87852b949b08790e99c257e6cc2 2024-11-25T10:58:16,583 INFO [M:0;6bfc37bad115:39013 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 23def87852b949b08790e99c257e6cc2 2024-11-25T10:58:16,583 INFO [M:0;6bfc37bad115:39013 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/23def87852b949b08790e99c257e6cc2, entries=156, sequenceid=2171, filesize=41.9 K 2024-11-25T10:58:16,584 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8df3eee21a2940d281883f6b86add61f as hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8df3eee21a2940d281883f6b86add61f 2024-11-25T10:58:16,587 INFO [M:0;6bfc37bad115:39013 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36023/user/jenkins/test-data/176cf969-7603-6cc1-bae8-36d929ff1770/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8df3eee21a2940d281883f6b86add61f, entries=1, sequenceid=2171, filesize=5.0 K 2024-11-25T10:58:16,588 INFO [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(3140): Finished flush of dataSize ~773.33 KB/791886, heapSize ~947.92 KB/970672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 527ms, sequenceid=2171, compaction requested=false 2024-11-25T10:58:16,601 INFO [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T10:58:16,601 DEBUG [M:0;6bfc37bad115:39013 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732532296061Disabling compacts and flushes for region at 1732532296061Disabling writes for close at 1732532296061Obtaining lock to block concurrent updates at 1732532296061Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732532296061Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=791886, getHeapSize=970912, getOffHeapSize=0, getCellsCount=2352 at 1732532296062 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732532296062Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732532296062Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732532296084 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732532296084Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732532296499 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732532296521 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732532296521Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732532296547 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732532296568 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732532296568Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cffb131: reopening flushed file at 1732532296575 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43316a2: reopening flushed file at 1732532296580 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3240689a: reopening flushed file at 1732532296583 (+3 ms)Finished flush of dataSize ~773.33 KB/791886, heapSize ~947.92 KB/970672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 527ms, sequenceid=2171, compaction requested=false at 1732532296588 (+5 ms)Writing region close event to WAL at 1732532296600 (+12 ms)Closed at 1732532296601 (+1 ms) 2024-11-25T10:58:16,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37429 is added to blk_1073741830_1006 (size=934144) 2024-11-25T10:58:16,607 INFO [M:0;6bfc37bad115:39013 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T10:58:16,607 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T10:58:16,607 INFO [M:0;6bfc37bad115:39013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39013 2024-11-25T10:58:16,607 INFO [M:0;6bfc37bad115:39013 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T10:58:16,709 INFO [M:0;6bfc37bad115:39013 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T10:58:16,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T10:58:16,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39013-0x10076940c1b0000, quorum=127.0.0.1:57296, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T10:58:16,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T10:58:16,724 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T10:58:16,724 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T10:58:16,724 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T10:58:16,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/hadoop.log.dir/,STOPPED} 2024-11-25T10:58:16,730 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T10:58:16,730 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T10:58:16,730 WARN [BP-161473846-172.17.0.3-1732532103561 heartbeating to localhost/127.0.0.1:36023 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T10:58:16,730 WARN [BP-161473846-172.17.0.3-1732532103561 heartbeating to localhost/127.0.0.1:36023 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-161473846-172.17.0.3-1732532103561 (Datanode Uuid 2cdee2b9-c7d3-4ba4-9102-a7bc63c6d504) service to localhost/127.0.0.1:36023 2024-11-25T10:58:16,734 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/data/data1/current/BP-161473846-172.17.0.3-1732532103561 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T10:58:16,734 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/cluster_39370e84-d197-90e8-5f79-3d44c352b009/data/data2/current/BP-161473846-172.17.0.3-1732532103561 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T10:58:16,736 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T10:58:16,748 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T10:58:16,749 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T10:58:16,749 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T10:58:16,749 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T10:58:16,749 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15127eac-3c5a-ddbb-b86a-66c08a539253/hadoop.log.dir/,STOPPED} 2024-11-25T10:58:16,779 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T10:58:16,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down